2024-11-22 17:14:08,876 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-22 17:14:08,887 main DEBUG Took 0.009337 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 17:14:08,887 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 17:14:08,887 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 17:14:08,888 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 17:14:08,889 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 17:14:08,896 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 17:14:08,908 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,909 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 17:14:08,909 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 17:14:08,910 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,911 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 17:14:08,912 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,912 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 17:14:08,912 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,913 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 17:14:08,913 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,914 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 17:14:08,914 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,915 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 17:14:08,915 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,915 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 17:14:08,916 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,916 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 17:14:08,917 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,917 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 17:14:08,917 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,918 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 17:14:08,918 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,918 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 17:14:08,919 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,919 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 17:14:08,920 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 17:14:08,921 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 17:14:08,923 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 17:14:08,924 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 17:14:08,925 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 17:14:08,925 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 17:14:08,933 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 17:14:08,935 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 17:14:08,937 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 17:14:08,937 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 17:14:08,938 main DEBUG createAppenders(={Console}) 2024-11-22 17:14:08,939 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-22 17:14:08,939 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-22 17:14:08,939 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-22 17:14:08,940 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 17:14:08,940 main DEBUG OutputStream closed 2024-11-22 17:14:08,940 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 17:14:08,941 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 17:14:08,941 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-22 17:14:09,011 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 17:14:09,013 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 17:14:09,014 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 17:14:09,015 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 17:14:09,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 17:14:09,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 17:14:09,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 17:14:09,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 17:14:09,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 17:14:09,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 17:14:09,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 17:14:09,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 17:14:09,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 17:14:09,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 17:14:09,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 17:14:09,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 17:14:09,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 17:14:09,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 17:14:09,023 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 17:14:09,023 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-22 17:14:09,023 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 17:14:09,024 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-22T17:14:09,247 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374 2024-11-22 17:14:09,250 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 17:14:09,250 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T17:14:09,258 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-22T17:14:09,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-22T17:14:09,294 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/cluster_c396a244-2610-d741-083f-3e053f26e882, deleteOnExit=true 2024-11-22T17:14:09,295 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-22T17:14:09,296 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/test.cache.data in system properties and HBase conf 2024-11-22T17:14:09,297 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/hadoop.tmp.dir in system properties and HBase conf 2024-11-22T17:14:09,297 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/hadoop.log.dir in system properties and HBase conf 2024-11-22T17:14:09,298 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-22T17:14:09,299 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-22T17:14:09,299 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-22T17:14:09,384 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T17:14:09,476 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T17:14:09,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-22T17:14:09,482 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-22T17:14:09,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-22T17:14:09,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T17:14:09,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-22T17:14:09,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-22T17:14:09,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T17:14:09,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T17:14:09,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-22T17:14:09,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/nfs.dump.dir in system properties and HBase conf 2024-11-22T17:14:09,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/java.io.tmpdir in system properties and HBase conf 2024-11-22T17:14:09,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T17:14:09,489 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-22T17:14:09,489 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-22T17:14:10,560 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-22T17:14:10,635 INFO [Time-limited test {}] log.Log(170): Logging initialized @2465ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-22T17:14:10,710 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T17:14:10,771 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T17:14:10,794 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T17:14:10,794 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T17:14:10,795 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-22T17:14:10,806 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T17:14:10,809 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/hadoop.log.dir/,AVAILABLE} 2024-11-22T17:14:10,810 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T17:14:10,991 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/java.io.tmpdir/jetty-localhost-37193-hadoop-hdfs-3_4_1-tests_jar-_-any-15769683858732542947/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T17:14:10,997 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:37193} 2024-11-22T17:14:10,998 INFO [Time-limited test {}] server.Server(415): Started @2829ms 2024-11-22T17:14:11,718 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T17:14:11,730 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T17:14:11,732 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T17:14:11,733 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T17:14:11,733 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-22T17:14:11,734 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4828e07b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/hadoop.log.dir/,AVAILABLE} 2024-11-22T17:14:11,735 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T17:14:11,839 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@72835e42{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/java.io.tmpdir/jetty-localhost-33873-hadoop-hdfs-3_4_1-tests_jar-_-any-1822134213357777815/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T17:14:11,840 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2098bdc7{HTTP/1.1, (http/1.1)}{localhost:33873} 2024-11-22T17:14:11,840 INFO [Time-limited test {}] server.Server(415): Started @3671ms 2024-11-22T17:14:11,887 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-22T17:14:13,368 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/cluster_c396a244-2610-d741-083f-3e053f26e882/data/data2/current/BP-932480829-172.17.0.2-1732295650022/current, will proceed with Du for space computation calculation, 2024-11-22T17:14:13,368 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/cluster_c396a244-2610-d741-083f-3e053f26e882/data/data1/current/BP-932480829-172.17.0.2-1732295650022/current, will proceed with Du for space computation calculation, 2024-11-22T17:14:13,401 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-22T17:14:13,449 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe9b9abed8b4db956 with lease ID 0x4cefde1b36a4eceb: Processing first storage report for DS-ecfed420-5116-44ce-b1ee-4bc101be089e from datanode DatanodeRegistration(127.0.0.1:35457, datanodeUuid=355a9353-cd06-4c0f-a5c5-96dfa7221a50, infoPort=42675, infoSecurePort=0, ipcPort=46601, storageInfo=lv=-57;cid=testClusterID;nsid=1565219990;c=1732295650022) 2024-11-22T17:14:13,450 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe9b9abed8b4db956 with lease ID 0x4cefde1b36a4eceb: from storage DS-ecfed420-5116-44ce-b1ee-4bc101be089e node DatanodeRegistration(127.0.0.1:35457, datanodeUuid=355a9353-cd06-4c0f-a5c5-96dfa7221a50, infoPort=42675, infoSecurePort=0, ipcPort=46601, storageInfo=lv=-57;cid=testClusterID;nsid=1565219990;c=1732295650022), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-22T17:14:13,451 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe9b9abed8b4db956 with lease ID 0x4cefde1b36a4eceb: Processing first storage report for DS-6ec9cf75-a52d-43eb-8124-5898332a5b56 from datanode DatanodeRegistration(127.0.0.1:35457, datanodeUuid=355a9353-cd06-4c0f-a5c5-96dfa7221a50, infoPort=42675, infoSecurePort=0, ipcPort=46601, storageInfo=lv=-57;cid=testClusterID;nsid=1565219990;c=1732295650022) 2024-11-22T17:14:13,451 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe9b9abed8b4db956 with lease ID 0x4cefde1b36a4eceb: from storage DS-6ec9cf75-a52d-43eb-8124-5898332a5b56 node DatanodeRegistration(127.0.0.1:35457, datanodeUuid=355a9353-cd06-4c0f-a5c5-96dfa7221a50, infoPort=42675, infoSecurePort=0, ipcPort=46601, storageInfo=lv=-57;cid=testClusterID;nsid=1565219990;c=1732295650022), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-22T17:14:13,517 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374 2024-11-22T17:14:13,604 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/cluster_c396a244-2610-d741-083f-3e053f26e882/zookeeper_0, clientPort=64218, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/cluster_c396a244-2610-d741-083f-3e053f26e882/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/cluster_c396a244-2610-d741-083f-3e053f26e882/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T17:14:13,616 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64218 2024-11-22T17:14:13,626 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T17:14:13,629 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T17:14:13,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741825_1001 (size=7) 2024-11-22T17:14:14,273 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8 with version=8 2024-11-22T17:14:14,274 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/hbase-staging 2024-11-22T17:14:14,361 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T17:14:14,562 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e61df7af61d2:0 server-side Connection retries=45 2024-11-22T17:14:14,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T17:14:14,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T17:14:14,576 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T17:14:14,576 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T17:14:14,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T17:14:14,706 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T17:14:14,761 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T17:14:14,770 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T17:14:14,773 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T17:14:14,797 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4695 (auto-detected) 2024-11-22T17:14:14,798 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T17:14:14,815 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45693 2024-11-22T17:14:14,837 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45693 connecting to ZooKeeper ensemble=127.0.0.1:64218 2024-11-22T17:14:15,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:456930x0, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T17:14:15,004 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45693-0x1016389f4780000 connected 2024-11-22T17:14:15,512 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T17:14:15,515 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T17:14:15,524 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T17:14:15,527 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8, hbase.cluster.distributed=false 2024-11-22T17:14:15,548 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T17:14:15,552 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45693 2024-11-22T17:14:15,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45693 2024-11-22T17:14:15,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45693 2024-11-22T17:14:15,554 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45693 2024-11-22T17:14:15,554 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45693 2024-11-22T17:14:15,658 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e61df7af61d2:0 server-side Connection retries=45 2024-11-22T17:14:15,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T17:14:15,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T17:14:15,660 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T17:14:15,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T17:14:15,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T17:14:15,663 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T17:14:15,665 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T17:14:15,666 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42253 2024-11-22T17:14:15,669 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42253 connecting to ZooKeeper ensemble=127.0.0.1:64218 2024-11-22T17:14:15,671 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T17:14:15,674 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T17:14:15,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422530x0, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T17:14:15,698 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42253-0x1016389f4780001 connected 2024-11-22T17:14:15,699 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T17:14:15,704 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T17:14:15,712 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T17:14:15,714 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T17:14:15,719 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T17:14:15,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42253 2024-11-22T17:14:15,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42253 2024-11-22T17:14:15,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42253 2024-11-22T17:14:15,721 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42253 2024-11-22T17:14:15,722 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42253 2024-11-22T17:14:15,737 DEBUG [M:0;e61df7af61d2:45693 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e61df7af61d2:45693 2024-11-22T17:14:15,738 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e61df7af61d2,45693,1732295654407 2024-11-22T17:14:15,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T17:14:15,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T17:14:15,769 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e61df7af61d2,45693,1732295654407 2024-11-22T17:14:15,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T17:14:15,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:14:15,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:14:15,864 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T17:14:15,866 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e61df7af61d2,45693,1732295654407 from backup master directory 2024-11-22T17:14:15,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T17:14:15,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e61df7af61d2,45693,1732295654407 2024-11-22T17:14:15,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T17:14:15,889 WARN [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T17:14:15,890 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e61df7af61d2,45693,1732295654407 2024-11-22T17:14:15,893 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T17:14:15,895 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T17:14:15,948 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/hbase.id] with ID: 10112fb3-27bd-4534-a278-6e2e2aff8d5f 2024-11-22T17:14:15,949 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/.tmp/hbase.id 2024-11-22T17:14:15,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741826_1002 (size=42) 2024-11-22T17:14:15,961 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/.tmp/hbase.id]:[hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/hbase.id] 2024-11-22T17:14:16,006 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T17:14:16,012 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T17:14:16,031 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-22T17:14:16,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:14:16,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:14:16,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741827_1003 (size=196) 2024-11-22T17:14:16,085 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T17:14:16,089 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T17:14:16,105 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:16,109 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T17:14:16,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741828_1004 (size=1189) 2024-11-22T17:14:16,560 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store 2024-11-22T17:14:16,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741829_1005 (size=34) 2024-11-22T17:14:16,987 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T17:14:16,993 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:14:16,994 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T17:14:16,994 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T17:14:16,994 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T17:14:16,996 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T17:14:16,996 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T17:14:16,996 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T17:14:16,997 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732295656994Disabling compacts and flushes for region at 1732295656994Disabling writes for close at 1732295656996 (+2 ms)Writing region close event to WAL at 1732295656996Closed at 1732295656996 2024-11-22T17:14:16,999 WARN [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/.initializing 2024-11-22T17:14:16,999 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/WALs/e61df7af61d2,45693,1732295654407 2024-11-22T17:14:17,006 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T17:14:17,021 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C45693%2C1732295654407, suffix=, logDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/WALs/e61df7af61d2,45693,1732295654407, archiveDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/oldWALs, maxLogs=10 2024-11-22T17:14:17,049 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/WALs/e61df7af61d2,45693,1732295654407/e61df7af61d2%2C45693%2C1732295654407.1732295657025, exclude list is [], retry=0 2024-11-22T17:14:17,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35457,DS-ecfed420-5116-44ce-b1ee-4bc101be089e,DISK] 2024-11-22T17:14:17,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-22T17:14:17,101 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/WALs/e61df7af61d2,45693,1732295654407/e61df7af61d2%2C45693%2C1732295654407.1732295657025 2024-11-22T17:14:17,102 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42675:42675)] 2024-11-22T17:14:17,103 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T17:14:17,103 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:14:17,106 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T17:14:17,107 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T17:14:17,144 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T17:14:17,170 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T17:14:17,173 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:17,175 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T17:14:17,176 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T17:14:17,180 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T17:14:17,180 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:17,181 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:14:17,181 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T17:14:17,184 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T17:14:17,185 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:17,186 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:14:17,186 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T17:14:17,189 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T17:14:17,190 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:17,191 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:14:17,191 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T17:14:17,195 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T17:14:17,196 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T17:14:17,202 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T17:14:17,203 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T17:14:17,207 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T17:14:17,212 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T17:14:17,218 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T17:14:17,219 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61049241, jitterRate=-0.09029541909694672}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T17:14:17,228 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732295657119Initializing all the Stores at 1732295657122 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732295657122Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295657123 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295657123Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295657123Cleaning up temporary data from old regions at 1732295657203 (+80 ms)Region opened successfully at 1732295657228 (+25 ms) 2024-11-22T17:14:17,230 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T17:14:17,273 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24390090, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0 2024-11-22T17:14:17,311 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T17:14:17,323 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T17:14:17,323 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T17:14:17,326 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T17:14:17,327 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T17:14:17,332 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T17:14:17,332 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T17:14:17,364 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T17:14:17,373 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T17:14:17,429 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T17:14:17,432 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T17:14:17,434 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T17:14:17,444 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T17:14:17,447 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T17:14:17,450 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T17:14:17,461 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T17:14:17,463 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T17:14:17,475 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T17:14:17,495 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T17:14:17,507 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T17:14:17,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T17:14:17,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T17:14:17,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:14:17,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:14:17,523 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e61df7af61d2,45693,1732295654407, sessionid=0x1016389f4780000, setting cluster-up flag (Was=false) 2024-11-22T17:14:17,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:14:17,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:14:17,591 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T17:14:17,594 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,45693,1732295654407 2024-11-22T17:14:17,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:14:17,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:14:17,655 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T17:14:17,659 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,45693,1732295654407 2024-11-22T17:14:17,669 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T17:14:17,727 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(746): ClusterId : 10112fb3-27bd-4534-a278-6e2e2aff8d5f 2024-11-22T17:14:17,729 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T17:14:17,737 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T17:14:17,747 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T17:14:17,747 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T17:14:17,748 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T17:14:17,754 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T17:14:17,762 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T17:14:17,759 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e61df7af61d2,45693,1732295654407 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T17:14:17,762 DEBUG [RS:0;e61df7af61d2:42253 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4444a7b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0 2024-11-22T17:14:17,767 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5 2024-11-22T17:14:17,767 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5 2024-11-22T17:14:17,767 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5 2024-11-22T17:14:17,767 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5 2024-11-22T17:14:17,768 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e61df7af61d2:0, corePoolSize=10, maxPoolSize=10 2024-11-22T17:14:17,768 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1 2024-11-22T17:14:17,768 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2 2024-11-22T17:14:17,768 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1 2024-11-22T17:14:17,772 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732295687772 2024-11-22T17:14:17,774 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T17:14:17,774 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T17:14:17,774 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T17:14:17,775 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T17:14:17,779 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T17:14:17,779 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T17:14:17,779 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e61df7af61d2:42253 2024-11-22T17:14:17,779 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T17:14:17,779 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T17:14:17,780 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:17,781 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T17:14:17,780 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,782 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T17:14:17,782 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T17:14:17,783 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T17:14:17,783 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T17:14:17,784 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T17:14:17,784 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T17:14:17,785 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(2659): reportForDuty to master=e61df7af61d2,45693,1732295654407 with port=42253, startcode=1732295655623 2024-11-22T17:14:17,786 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T17:14:17,787 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T17:14:17,788 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732295657788,5,FailOnTimeoutGroup] 2024-11-22T17:14:17,789 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732295657789,5,FailOnTimeoutGroup] 2024-11-22T17:14:17,789 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741831_1007 (size=1321) 2024-11-22T17:14:17,790 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T17:14:17,790 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,791 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,793 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T17:14:17,793 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8 2024-11-22T17:14:17,798 DEBUG [RS:0;e61df7af61d2:42253 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T17:14:17,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741832_1008 (size=32) 2024-11-22T17:14:17,829 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44925, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T17:14:17,835 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45693 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e61df7af61d2,42253,1732295655623 2024-11-22T17:14:17,837 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45693 {}] master.ServerManager(517): Registering regionserver=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:17,849 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8 2024-11-22T17:14:17,849 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40221 2024-11-22T17:14:17,849 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T17:14:17,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T17:14:17,862 DEBUG [RS:0;e61df7af61d2:42253 {}] zookeeper.ZKUtil(111): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e61df7af61d2,42253,1732295655623 2024-11-22T17:14:17,862 WARN [RS:0;e61df7af61d2:42253 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T17:14:17,862 INFO [RS:0;e61df7af61d2:42253 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T17:14:17,863 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/WALs/e61df7af61d2,42253,1732295655623 2024-11-22T17:14:17,865 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e61df7af61d2,42253,1732295655623] 2024-11-22T17:14:17,891 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T17:14:17,907 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T17:14:17,911 INFO [RS:0;e61df7af61d2:42253 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T17:14:17,911 INFO [RS:0;e61df7af61d2:42253 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,912 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T17:14:17,917 INFO [RS:0;e61df7af61d2:42253 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T17:14:17,919 INFO [RS:0;e61df7af61d2:42253 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,919 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1 2024-11-22T17:14:17,919 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1 2024-11-22T17:14:17,919 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1 2024-11-22T17:14:17,920 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1 2024-11-22T17:14:17,920 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1 2024-11-22T17:14:17,920 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2 2024-11-22T17:14:17,920 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1 2024-11-22T17:14:17,920 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1 2024-11-22T17:14:17,920 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1 2024-11-22T17:14:17,920 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1 2024-11-22T17:14:17,921 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1 2024-11-22T17:14:17,921 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1 2024-11-22T17:14:17,921 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3 2024-11-22T17:14:17,921 DEBUG [RS:0;e61df7af61d2:42253 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3 2024-11-22T17:14:17,922 INFO [RS:0;e61df7af61d2:42253 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,922 INFO [RS:0;e61df7af61d2:42253 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,922 INFO [RS:0;e61df7af61d2:42253 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,922 INFO [RS:0;e61df7af61d2:42253 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,922 INFO [RS:0;e61df7af61d2:42253 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,922 INFO [RS:0;e61df7af61d2:42253 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,42253,1732295655623-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T17:14:17,941 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T17:14:17,942 INFO [RS:0;e61df7af61d2:42253 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,42253,1732295655623-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,943 INFO [RS:0;e61df7af61d2:42253 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,943 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.Replication(171): e61df7af61d2,42253,1732295655623 started 2024-11-22T17:14:17,960 INFO [RS:0;e61df7af61d2:42253 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:17,960 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1482): Serving as e61df7af61d2,42253,1732295655623, RpcServer on e61df7af61d2/172.17.0.2:42253, sessionid=0x1016389f4780001 2024-11-22T17:14:17,961 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T17:14:17,961 DEBUG [RS:0;e61df7af61d2:42253 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e61df7af61d2,42253,1732295655623 2024-11-22T17:14:17,962 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,42253,1732295655623' 2024-11-22T17:14:17,962 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T17:14:17,963 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T17:14:17,963 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T17:14:17,963 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T17:14:17,964 DEBUG [RS:0;e61df7af61d2:42253 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e61df7af61d2,42253,1732295655623 2024-11-22T17:14:17,964 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,42253,1732295655623' 2024-11-22T17:14:17,964 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T17:14:17,964 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T17:14:17,965 DEBUG [RS:0;e61df7af61d2:42253 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T17:14:17,965 INFO [RS:0;e61df7af61d2:42253 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T17:14:17,965 INFO [RS:0;e61df7af61d2:42253 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T17:14:18,071 INFO [RS:0;e61df7af61d2:42253 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T17:14:18,076 INFO [RS:0;e61df7af61d2:42253 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C42253%2C1732295655623, suffix=, logDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/WALs/e61df7af61d2,42253,1732295655623, archiveDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/oldWALs, maxLogs=32 2024-11-22T17:14:18,091 DEBUG [RS:0;e61df7af61d2:42253 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/WALs/e61df7af61d2,42253,1732295655623/e61df7af61d2%2C42253%2C1732295655623.1732295658078, exclude list is [], retry=0 2024-11-22T17:14:18,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35457,DS-ecfed420-5116-44ce-b1ee-4bc101be089e,DISK] 2024-11-22T17:14:18,098 INFO [RS:0;e61df7af61d2:42253 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/WALs/e61df7af61d2,42253,1732295655623/e61df7af61d2%2C42253%2C1732295655623.1732295658078 2024-11-22T17:14:18,099 DEBUG [RS:0;e61df7af61d2:42253 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42675:42675)] 2024-11-22T17:14:18,205 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:14:18,208 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T17:14:18,210 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T17:14:18,210 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:18,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T17:14:18,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T17:14:18,214 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T17:14:18,214 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:18,215 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T17:14:18,215 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T17:14:18,218 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T17:14:18,218 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:18,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T17:14:18,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T17:14:18,222 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T17:14:18,222 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:18,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T17:14:18,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T17:14:18,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740 2024-11-22T17:14:18,226 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740 2024-11-22T17:14:18,229 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T17:14:18,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T17:14:18,231 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T17:14:18,233 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T17:14:18,237 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T17:14:18,238 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70748926, jitterRate=0.05424115061759949}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T17:14:18,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732295658205Initializing all the Stores at 1732295658207 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732295658207Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732295658207Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295658207Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732295658207Cleaning up temporary data from old regions at 1732295658230 (+23 ms)Region opened successfully at 1732295658240 (+10 ms) 2024-11-22T17:14:18,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T17:14:18,241 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T17:14:18,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T17:14:18,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T17:14:18,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T17:14:18,242 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T17:14:18,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732295658241Disabling compacts and flushes for region at 1732295658241Disabling writes for close at 1732295658241Writing region close event to WAL at 1732295658242 (+1 ms)Closed at 1732295658242 2024-11-22T17:14:18,246 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T17:14:18,246 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T17:14:18,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T17:14:18,259 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T17:14:18,262 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T17:14:18,415 DEBUG [e61df7af61d2:45693 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-22T17:14:18,430 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:18,436 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,42253,1732295655623, state=OPENING 2024-11-22T17:14:18,482 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T17:14:18,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:14:18,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:14:18,495 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T17:14:18,495 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T17:14:18,498 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T17:14:18,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:14:18,680 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T17:14:18,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37393, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T17:14:18,695 INFO [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T17:14:18,696 INFO [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T17:14:18,696 INFO [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T17:14:18,700 INFO [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C42253%2C1732295655623.meta, suffix=.meta, logDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/WALs/e61df7af61d2,42253,1732295655623, archiveDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/oldWALs, maxLogs=32 2024-11-22T17:14:18,715 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/WALs/e61df7af61d2,42253,1732295655623/e61df7af61d2%2C42253%2C1732295655623.meta.1732295658702.meta, exclude list is [], retry=0 2024-11-22T17:14:18,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35457,DS-ecfed420-5116-44ce-b1ee-4bc101be089e,DISK] 2024-11-22T17:14:18,722 INFO [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/WALs/e61df7af61d2,42253,1732295655623/e61df7af61d2%2C42253%2C1732295655623.meta.1732295658702.meta 2024-11-22T17:14:18,722 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42675:42675)] 2024-11-22T17:14:18,723 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T17:14:18,724 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T17:14:18,727 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T17:14:18,731 INFO [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T17:14:18,735 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T17:14:18,736 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:14:18,736 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T17:14:18,736 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T17:14:18,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T17:14:18,740 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T17:14:18,741 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:18,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T17:14:18,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T17:14:18,744 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T17:14:18,744 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:18,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T17:14:18,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T17:14:18,746 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T17:14:18,747 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:18,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T17:14:18,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T17:14:18,749 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T17:14:18,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:18,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T17:14:18,751 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T17:14:18,752 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740 2024-11-22T17:14:18,755 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740 2024-11-22T17:14:18,757 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T17:14:18,758 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T17:14:18,758 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T17:14:18,761 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T17:14:18,763 INFO [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70458006, jitterRate=0.04990610480308533}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T17:14:18,763 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T17:14:18,764 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732295658737Writing region info on filesystem at 1732295658737Initializing all the Stores at 1732295658738 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732295658739 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732295658739Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295658739Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732295658739Cleaning up temporary data from old regions at 1732295658758 (+19 ms)Running coprocessor post-open hooks at 1732295658763 (+5 ms)Region opened successfully at 1732295658764 (+1 ms) 2024-11-22T17:14:18,771 INFO [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732295658671 2024-11-22T17:14:18,781 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T17:14:18,782 INFO [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T17:14:18,783 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:18,785 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,42253,1732295655623, state=OPEN 2024-11-22T17:14:18,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T17:14:18,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T17:14:18,894 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T17:14:18,894 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T17:14:18,894 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:18,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T17:14:18,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,42253,1732295655623 in 392 msec 2024-11-22T17:14:18,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T17:14:18,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 655 msec 2024-11-22T17:14:18,914 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T17:14:18,914 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T17:14:18,935 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:18,936 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:18,954 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:18,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54513, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:18,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2790 sec 2024-11-22T17:14:18,977 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732295658977, completionTime=-1 2024-11-22T17:14:18,979 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-22T17:14:18,980 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T17:14:19,004 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T17:14:19,005 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732295719005 2024-11-22T17:14:19,005 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732295779005 2024-11-22T17:14:19,005 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-22T17:14:19,007 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,45693,1732295654407-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:19,007 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,45693,1732295654407-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:19,007 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,45693,1732295654407-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:19,009 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e61df7af61d2:45693, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:19,009 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:19,010 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:19,016 DEBUG [master/e61df7af61d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T17:14:19,036 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.146sec 2024-11-22T17:14:19,038 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T17:14:19,039 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T17:14:19,040 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T17:14:19,040 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T17:14:19,040 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T17:14:19,041 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,45693,1732295654407-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T17:14:19,042 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,45693,1732295654407-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T17:14:19,051 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T17:14:19,051 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T17:14:19,052 INFO [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,45693,1732295654407-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T17:14:19,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8a5f59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:19,141 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T17:14:19,142 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T17:14:19,145 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:19,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:19,154 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c924b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:19,165 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:19,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@650c631a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:19,168 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:19,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:19,198 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e61df7af61d2,45693,1732295654407 2024-11-22T17:14:19,209 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=280, ProcessCount=11, AvailableMemoryMB=2739 2024-11-22T17:14:19,217 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T17:14:19,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e61df7af61d2,45693,1732295654407 2024-11-22T17:14:19,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54ffeb4e 2024-11-22T17:14:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T17:14:19,247 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T17:14:19,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T17:14:19,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T17:14:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T17:14:19,267 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T17:14:19,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-22T17:14:19,273 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:19,276 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T17:14:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T17:14:19,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741835_1011 (size=1061) 2024-11-22T17:14:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T17:14:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T17:14:19,696 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8 2024-11-22T17:14:19,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741836_1012 (size=53) 2024-11-22T17:14:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T17:14:20,110 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:14:20,111 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c70a0885ec0e4f41398ae6890c8dabe5, disabling compactions & flushes 2024-11-22T17:14:20,111 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:20,111 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:20,111 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. after waiting 0 ms 2024-11-22T17:14:20,111 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:20,111 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:20,112 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c70a0885ec0e4f41398ae6890c8dabe5: Waiting for close lock at 1732295660110Disabling compacts and flushes for region at 1732295660110Disabling writes for close at 1732295660111 (+1 ms)Writing region close event to WAL at 1732295660111Closed at 1732295660111 2024-11-22T17:14:20,117 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T17:14:20,123 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732295660117"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732295660117"}]},"ts":"1732295660117"} 2024-11-22T17:14:20,128 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T17:14:20,130 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T17:14:20,133 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295660130"}]},"ts":"1732295660130"} 2024-11-22T17:14:20,137 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T17:14:20,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c70a0885ec0e4f41398ae6890c8dabe5, ASSIGN}] 2024-11-22T17:14:20,141 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c70a0885ec0e4f41398ae6890c8dabe5, ASSIGN 2024-11-22T17:14:20,143 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c70a0885ec0e4f41398ae6890c8dabe5, ASSIGN; state=OFFLINE, location=e61df7af61d2,42253,1732295655623; forceNewPlan=false, retain=false 2024-11-22T17:14:20,294 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c70a0885ec0e4f41398ae6890c8dabe5, regionState=OPENING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:20,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c70a0885ec0e4f41398ae6890c8dabe5, ASSIGN because future has completed 2024-11-22T17:14:20,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:14:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T17:14:20,468 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:20,468 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} 2024-11-22T17:14:20,469 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:20,469 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:14:20,469 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:20,469 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:20,472 INFO [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:20,475 INFO [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:14:20,476 INFO [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c70a0885ec0e4f41398ae6890c8dabe5 columnFamilyName A 2024-11-22T17:14:20,476 DEBUG [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:20,477 INFO [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] regionserver.HStore(327): Store=c70a0885ec0e4f41398ae6890c8dabe5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:14:20,477 INFO [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:20,479 INFO [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:14:20,480 INFO [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c70a0885ec0e4f41398ae6890c8dabe5 columnFamilyName B 2024-11-22T17:14:20,480 DEBUG [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:20,480 INFO [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] regionserver.HStore(327): Store=c70a0885ec0e4f41398ae6890c8dabe5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:14:20,481 INFO [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:20,483 INFO [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:14:20,483 INFO [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c70a0885ec0e4f41398ae6890c8dabe5 columnFamilyName C 2024-11-22T17:14:20,483 DEBUG [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:20,484 INFO [StoreOpener-c70a0885ec0e4f41398ae6890c8dabe5-1 {}] regionserver.HStore(327): Store=c70a0885ec0e4f41398ae6890c8dabe5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:14:20,484 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:20,484 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:20,486 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:20,487 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:20,489 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:20,489 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:20,490 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T17:14:20,492 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:20,496 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T17:14:20,497 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c70a0885ec0e4f41398ae6890c8dabe5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75359383, jitterRate=0.12294231355190277}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T17:14:20,497 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:20,498 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c70a0885ec0e4f41398ae6890c8dabe5: Running coprocessor pre-open hook at 1732295660469Writing region info on filesystem at 1732295660469Initializing all the Stores at 1732295660471 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295660471Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295660472 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295660472Cleaning up temporary data from old regions at 1732295660489 (+17 ms)Running coprocessor post-open hooks at 1732295660497 (+8 ms)Region opened successfully at 1732295660497 2024-11-22T17:14:20,500 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., pid=6, masterSystemTime=1732295660459 2024-11-22T17:14:20,503 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:20,503 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:20,505 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c70a0885ec0e4f41398ae6890c8dabe5, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:20,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:14:20,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T17:14:20,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 in 209 msec 2024-11-22T17:14:20,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T17:14:20,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c70a0885ec0e4f41398ae6890c8dabe5, ASSIGN in 376 msec 2024-11-22T17:14:20,521 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T17:14:20,521 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295660521"}]},"ts":"1732295660521"} 2024-11-22T17:14:20,524 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T17:14:20,526 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T17:14:20,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2690 sec 2024-11-22T17:14:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T17:14:21,456 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:21,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e4ddb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:21,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:21,466 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c34a958, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:21,470 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:21,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127234f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,472 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:21,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:21,479 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T17:14:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e61df7af61d2,45693,1732295654407 2024-11-22T17:14:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@42827a35 2024-11-22T17:14:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T17:14:21,485 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T17:14:21,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@378e4d76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,490 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:21,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:21,492 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5367b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:21,496 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:21,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39deefb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aae1f3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:21,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:21,500 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ccbe83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:21,504 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ac8568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728f452, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:21,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:21,507 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec6ad86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:21,511 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c5f854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a822d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:21,514 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:21,515 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51268a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:21,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:21,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:21,519 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:21,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1768b9d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7813eeba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,521 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:21,521 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:21,522 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70343832, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:21,524 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5127c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,527 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f66d26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,527 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:21,527 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:21,528 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a9c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:21,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:21,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:21,531 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe9eb0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@225d1960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:21,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:21,534 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@796fbe9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:21,536 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74d388d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93b4e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:21,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:21,539 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c24d9c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:21,542 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f83cf40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7449cb3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:21,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:21,545 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20b2908d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:21,547 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:21,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cfd6776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:21,552 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:21,552 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:21,552 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:21,552 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:21,553 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:21,554 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:21,554 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:14:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:21,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:21,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:21,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:21,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:21,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:21,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:21,560 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-22T17:14:21,562 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:14:21,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:21,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:14:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:14:21,568 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:14:21,570 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:14:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:14:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:14:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:14:21,573 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:14:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:21,576 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:21,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:14:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:14:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:14:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T17:14:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:21,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T17:14:21,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:21,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:21,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:21,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:21,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:21,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T17:14:21,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T17:14:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:21,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:21,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:21,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:21,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:21,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295721769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295721771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:21,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/efb4a87b0a6644dc98085473cc58d2b4 is 50, key is test_row_0/A:col10/1732295661622/Put/seqid=0 2024-11-22T17:14:21,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295721772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:21,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295721792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:21,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295721800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:21,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:21,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741837_1013 (size=14341) 2024-11-22T17:14:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T17:14:21,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295721953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:21,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T17:14:21,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295721955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:21,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:21,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295721954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295721955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:21,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295721956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:21,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:21,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:21,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:22,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T17:14:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:22,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:22,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:22,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295722162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:22,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295722173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:22,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295722174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:22,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295722174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:22,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295722173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T17:14:22,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/efb4a87b0a6644dc98085473cc58d2b4 2024-11-22T17:14:22,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T17:14:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:22,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:22,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/913237a97906446bab93b1a2cc4d6f7e is 50, key is test_row_0/B:col10/1732295661622/Put/seqid=0 2024-11-22T17:14:22,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741838_1014 (size=12001) 2024-11-22T17:14:22,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/913237a97906446bab93b1a2cc4d6f7e 2024-11-22T17:14:22,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T17:14:22,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:22,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:22,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:22,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:22,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:22,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/950737c9a65d4188b03ea22b929cb2de is 50, key is test_row_0/C:col10/1732295661622/Put/seqid=0 2024-11-22T17:14:22,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741839_1015 (size=12001) 2024-11-22T17:14:22,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/950737c9a65d4188b03ea22b929cb2de 2024-11-22T17:14:22,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295722474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:22,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295722482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:22,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:22,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295722482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:22,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:22,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/efb4a87b0a6644dc98085473cc58d2b4 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/efb4a87b0a6644dc98085473cc58d2b4 2024-11-22T17:14:22,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295722492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:22,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295722493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:22,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/efb4a87b0a6644dc98085473cc58d2b4, entries=200, sequenceid=13, filesize=14.0 K 2024-11-22T17:14:22,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/913237a97906446bab93b1a2cc4d6f7e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/913237a97906446bab93b1a2cc4d6f7e 2024-11-22T17:14:22,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/913237a97906446bab93b1a2cc4d6f7e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-22T17:14:22,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/950737c9a65d4188b03ea22b929cb2de as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/950737c9a65d4188b03ea22b929cb2de 2024-11-22T17:14:22,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/950737c9a65d4188b03ea22b929cb2de, entries=150, sequenceid=13, filesize=11.7 K 2024-11-22T17:14:22,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c70a0885ec0e4f41398ae6890c8dabe5 in 931ms, sequenceid=13, compaction requested=false 2024-11-22T17:14:22,574 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T17:14:22,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:22,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T17:14:22,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:22,590 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T17:14:22,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:22,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:22,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:22,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:22,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:22,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:22,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/bf6918868f4f420ba8f19078abd6ef01 is 50, key is test_row_0/A:col10/1732295661761/Put/seqid=0 2024-11-22T17:14:22,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741840_1016 (size=12001) 2024-11-22T17:14:22,627 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/bf6918868f4f420ba8f19078abd6ef01 2024-11-22T17:14:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/baec977fffa446b69e2d0d879b70f3a9 is 50, key is test_row_0/B:col10/1732295661761/Put/seqid=0 2024-11-22T17:14:22,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741841_1017 (size=12001) 2024-11-22T17:14:22,673 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/baec977fffa446b69e2d0d879b70f3a9 2024-11-22T17:14:22,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/204c0af83a884769968b8783459f183b is 50, key is test_row_0/C:col10/1732295661761/Put/seqid=0 2024-11-22T17:14:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T17:14:22,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741842_1018 (size=12001) 2024-11-22T17:14:22,732 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/204c0af83a884769968b8783459f183b 2024-11-22T17:14:22,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/bf6918868f4f420ba8f19078abd6ef01 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/bf6918868f4f420ba8f19078abd6ef01 2024-11-22T17:14:22,779 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/bf6918868f4f420ba8f19078abd6ef01, entries=150, sequenceid=33, filesize=11.7 K 2024-11-22T17:14:22,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/baec977fffa446b69e2d0d879b70f3a9 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/baec977fffa446b69e2d0d879b70f3a9 2024-11-22T17:14:22,825 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/baec977fffa446b69e2d0d879b70f3a9, entries=150, sequenceid=33, filesize=11.7 K 2024-11-22T17:14:22,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/204c0af83a884769968b8783459f183b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/204c0af83a884769968b8783459f183b 2024-11-22T17:14:22,854 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/204c0af83a884769968b8783459f183b, entries=150, sequenceid=33, filesize=11.7 K 2024-11-22T17:14:22,856 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for c70a0885ec0e4f41398ae6890c8dabe5 in 267ms, sequenceid=33, compaction requested=false 2024-11-22T17:14:22,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:22,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:22,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-22T17:14:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-22T17:14:22,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T17:14:22,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2830 sec 2024-11-22T17:14:22,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.3130 sec 2024-11-22T17:14:23,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:14:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:23,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/6f486ddcad2b44cebacf6de8fa067cad is 50, key is test_row_0/A:col10/1732295663000/Put/seqid=0 2024-11-22T17:14:23,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741843_1019 (size=14341) 2024-11-22T17:14:23,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/6f486ddcad2b44cebacf6de8fa067cad 2024-11-22T17:14:23,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/a1247f6fcacf49d0a0242ef53a40710b is 50, key is test_row_0/B:col10/1732295663000/Put/seqid=0 2024-11-22T17:14:23,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295663069, max timestamp=1732295663096Num uniques 859; 2024-11-22T17:14:23,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295663069, max timestamp=1732295663095Num uniques 859; 2024-11-22T17:14:23,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:23,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295663069, max timestamp=1732295663096Num uniques 859; 2024-11-22T17:14:23,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:23,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:23,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741844_1020 (size=12001) 2024-11-22T17:14:23,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/a1247f6fcacf49d0a0242ef53a40710b 2024-11-22T17:14:23,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/65964acb58ae488dae2b38d623dad561 is 50, key is test_row_0/C:col10/1732295663000/Put/seqid=0 2024-11-22T17:14:23,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295723157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295723159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741845_1021 (size=12001) 2024-11-22T17:14:23,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295723160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/65964acb58ae488dae2b38d623dad561 2024-11-22T17:14:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295723164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295723168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/6f486ddcad2b44cebacf6de8fa067cad as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/6f486ddcad2b44cebacf6de8fa067cad 2024-11-22T17:14:23,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/6f486ddcad2b44cebacf6de8fa067cad, entries=200, sequenceid=43, filesize=14.0 K 2024-11-22T17:14:23,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/a1247f6fcacf49d0a0242ef53a40710b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/a1247f6fcacf49d0a0242ef53a40710b 2024-11-22T17:14:23,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/a1247f6fcacf49d0a0242ef53a40710b, entries=150, sequenceid=43, filesize=11.7 K 2024-11-22T17:14:23,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/65964acb58ae488dae2b38d623dad561 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/65964acb58ae488dae2b38d623dad561 2024-11-22T17:14:23,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/65964acb58ae488dae2b38d623dad561, entries=150, sequenceid=43, filesize=11.7 K 2024-11-22T17:14:23,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c70a0885ec0e4f41398ae6890c8dabe5 in 245ms, sequenceid=43, compaction requested=true 2024-11-22T17:14:23,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:23,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:14:23,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:23,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:14:23,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:23,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:14:23,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:23,274 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:14:23,277 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:14:23,278 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:14:23,280 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:14:23,280 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/B is initiating minor compaction (all files) 2024-11-22T17:14:23,280 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/A is initiating minor compaction (all files) 2024-11-22T17:14:23,281 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/A in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:23,281 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/B in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:23,282 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/efb4a87b0a6644dc98085473cc58d2b4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/bf6918868f4f420ba8f19078abd6ef01, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/6f486ddcad2b44cebacf6de8fa067cad] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=39.7 K 2024-11-22T17:14:23,282 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/913237a97906446bab93b1a2cc4d6f7e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/baec977fffa446b69e2d0d879b70f3a9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/a1247f6fcacf49d0a0242ef53a40710b] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=35.2 K 2024-11-22T17:14:23,284 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 913237a97906446bab93b1a2cc4d6f7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732295661622 2024-11-22T17:14:23,284 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting efb4a87b0a6644dc98085473cc58d2b4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732295661580 2024-11-22T17:14:23,285 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf6918868f4f420ba8f19078abd6ef01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732295661761 2024-11-22T17:14:23,285 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting baec977fffa446b69e2d0d879b70f3a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732295661761 2024-11-22T17:14:23,286 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f486ddcad2b44cebacf6de8fa067cad, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732295663000 2024-11-22T17:14:23,286 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a1247f6fcacf49d0a0242ef53a40710b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732295663000 2024-11-22T17:14:23,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-22T17:14:23,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:23,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:23,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:23,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:23,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:23,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:23,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/9594292a1792425bbe77baf95b9756f1 is 50, key is test_row_0/A:col10/1732295663159/Put/seqid=0 2024-11-22T17:14:23,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295723314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,330 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#A#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:23,331 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/c6e5b27aa6014a998e3b31aa4d56e312 is 50, key is test_row_0/A:col10/1732295663000/Put/seqid=0 2024-11-22T17:14:23,332 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#B#compaction#11 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:23,334 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/82680d69cf6e43afa382cfa67fb7fc98 is 50, key is test_row_0/B:col10/1732295663000/Put/seqid=0 2024-11-22T17:14:23,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295723319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295723320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295723323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295723325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741846_1022 (size=14341) 2024-11-22T17:14:23,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741847_1023 (size=12104) 2024-11-22T17:14:23,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741848_1024 (size=12104) 2024-11-22T17:14:23,391 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/82680d69cf6e43afa382cfa67fb7fc98 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/82680d69cf6e43afa382cfa67fb7fc98 2024-11-22T17:14:23,419 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/B of c70a0885ec0e4f41398ae6890c8dabe5 into 82680d69cf6e43afa382cfa67fb7fc98(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:23,419 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:23,422 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/B, priority=13, startTime=1732295663271; duration=0sec 2024-11-22T17:14:23,423 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:23,423 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:B 2024-11-22T17:14:23,424 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:14:23,426 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:14:23,427 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/C is initiating minor compaction (all files) 2024-11-22T17:14:23,427 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/C in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:23,427 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/950737c9a65d4188b03ea22b929cb2de, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/204c0af83a884769968b8783459f183b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/65964acb58ae488dae2b38d623dad561] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=35.2 K 2024-11-22T17:14:23,429 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 950737c9a65d4188b03ea22b929cb2de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732295661622 2024-11-22T17:14:23,430 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 204c0af83a884769968b8783459f183b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732295661761 2024-11-22T17:14:23,430 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 65964acb58ae488dae2b38d623dad561, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732295663000 2024-11-22T17:14:23,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295723432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295723441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295723442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295723452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295723452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,464 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:23,465 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/c38071bb8116403c8e2c03b2c207085d is 50, key is test_row_0/C:col10/1732295663000/Put/seqid=0 2024-11-22T17:14:23,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741849_1025 (size=12104) 2024-11-22T17:14:23,513 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/c38071bb8116403c8e2c03b2c207085d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/c38071bb8116403c8e2c03b2c207085d 2024-11-22T17:14:23,532 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/C of c70a0885ec0e4f41398ae6890c8dabe5 into c38071bb8116403c8e2c03b2c207085d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:23,532 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:23,533 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/C, priority=13, startTime=1732295663271; duration=0sec 2024-11-22T17:14:23,533 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:23,533 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:C 2024-11-22T17:14:23,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295723642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295723653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295723654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295723662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295723662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T17:14:23,733 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:23,733 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:14:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-22T17:14:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T17:14:23,740 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:14:23,742 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:14:23,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:14:23,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/9594292a1792425bbe77baf95b9756f1 2024-11-22T17:14:23,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/7eaf27ef7f1548e583e2a3a2dc6dbc41 is 50, key is test_row_0/B:col10/1732295663159/Put/seqid=0 2024-11-22T17:14:23,780 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/c6e5b27aa6014a998e3b31aa4d56e312 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/c6e5b27aa6014a998e3b31aa4d56e312 2024-11-22T17:14:23,797 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/A of c70a0885ec0e4f41398ae6890c8dabe5 into c6e5b27aa6014a998e3b31aa4d56e312(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:23,797 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:23,797 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/A, priority=13, startTime=1732295663268; duration=0sec 2024-11-22T17:14:23,798 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:23,798 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:A 2024-11-22T17:14:23,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741850_1026 (size=12001) 2024-11-22T17:14:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T17:14:23,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T17:14:23,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:23,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:23,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:23,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:23,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:23,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295723952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295723961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295723972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295723982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295723982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T17:14:24,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T17:14:24,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:24,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:24,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:24,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:24,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:24,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/7eaf27ef7f1548e583e2a3a2dc6dbc41 2024-11-22T17:14:24,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T17:14:24,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:24,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:24,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:24,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:24,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:24,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/91bf32e73c7f4e71b255e96b95d06956 is 50, key is test_row_0/C:col10/1732295663159/Put/seqid=0 2024-11-22T17:14:24,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741851_1027 (size=12001) 2024-11-22T17:14:24,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/91bf32e73c7f4e71b255e96b95d06956 2024-11-22T17:14:24,252 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T17:14:24,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/9594292a1792425bbe77baf95b9756f1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/9594292a1792425bbe77baf95b9756f1 2024-11-22T17:14:24,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/9594292a1792425bbe77baf95b9756f1, entries=200, sequenceid=80, filesize=14.0 K 2024-11-22T17:14:24,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/7eaf27ef7f1548e583e2a3a2dc6dbc41 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/7eaf27ef7f1548e583e2a3a2dc6dbc41 2024-11-22T17:14:24,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/7eaf27ef7f1548e583e2a3a2dc6dbc41, entries=150, sequenceid=80, filesize=11.7 K 2024-11-22T17:14:24,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/91bf32e73c7f4e71b255e96b95d06956 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/91bf32e73c7f4e71b255e96b95d06956 2024-11-22T17:14:24,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/91bf32e73c7f4e71b255e96b95d06956, entries=150, sequenceid=80, filesize=11.7 K 2024-11-22T17:14:24,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for c70a0885ec0e4f41398ae6890c8dabe5 in 1041ms, sequenceid=80, compaction requested=false 2024-11-22T17:14:24,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:24,350 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T17:14:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T17:14:24,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T17:14:24,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:24,369 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-22T17:14:24,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:24,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:24,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:24,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:24,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:24,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:24,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/59fd11c2b9034f679e80a8f6960d57ad is 50, key is test_row_0/A:col10/1732295663313/Put/seqid=0 2024-11-22T17:14:24,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741852_1028 (size=12001) 2024-11-22T17:14:24,428 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/59fd11c2b9034f679e80a8f6960d57ad 2024-11-22T17:14:24,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/4a1cf233faad4279a5905af44a32a471 is 50, key is test_row_0/B:col10/1732295663313/Put/seqid=0 2024-11-22T17:14:24,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741853_1029 (size=12001) 2024-11-22T17:14:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:24,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:24,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295664513, max timestamp=1732295664532Num uniques 859; 2024-11-22T17:14:24,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:24,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295664513, max timestamp=1732295664532Num uniques 859; 2024-11-22T17:14:24,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:24,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295664513, max timestamp=1732295664532Num uniques 859; 2024-11-22T17:14:24,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:24,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295724655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:24,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12262, totalHeapSize=28112, min timestamp=1732295664559, max timestamp=1732295664663Num uniques 1018; 2024-11-22T17:14:24,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46612, new segment data size=12262, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-22T17:14:24,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732295664559, max timestamp=1732295664663Num uniques 1018; 2024-11-22T17:14:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:24,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-22T17:14:24,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732295664559, max timestamp=1732295664662Num uniques 1018; 2024-11-22T17:14:24,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-22T17:14:24,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:24,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295724727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295724727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:24,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295724748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:24,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732295664694, max timestamp=1732295664742Num uniques 1127; 2024-11-22T17:14:24,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732295664694, max timestamp=1732295664742Num uniques 1127; 2024-11-22T17:14:24,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732295664694, max timestamp=1732295664742Num uniques 1027; 2024-11-22T17:14:24,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-22T17:14:24,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-22T17:14:24,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-22T17:14:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:24,758 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-22T17:14:24,759 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-22T17:14:24,762 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T17:14:24,762 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-22T17:14:24,762 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-22T17:14:24,762 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-22T17:14:24,764 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T17:14:24,764 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-22T17:14:24,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732295664825, max timestamp=1732295664844Num uniques 1036; 2024-11-22T17:14:24,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732295664825, max timestamp=1732295664844Num uniques 1036; 2024-11-22T17:14:24,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732295664825, max timestamp=1732295664844Num uniques 1036; 2024-11-22T17:14:24,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-22T17:14:24,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-22T17:14:24,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-22T17:14:24,866 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/4a1cf233faad4279a5905af44a32a471 2024-11-22T17:14:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T17:14:24,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/d77ae0c438664a5da80efd64b433772c is 50, key is test_row_0/C:col10/1732295663313/Put/seqid=0 2024-11-22T17:14:24,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741854_1030 (size=12001) 2024-11-22T17:14:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295724894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:24,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1732295664884, max timestamp=1732295664906Num uniques 1095; 2024-11-22T17:14:24,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-22T17:14:24,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1732295664884, max timestamp=1732295664906Num uniques 1095; 2024-11-22T17:14:24,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1732295664884, max timestamp=1732295664906Num uniques 1095; 2024-11-22T17:14:24,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-22T17:14:24,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-22T17:14:24,914 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/d77ae0c438664a5da80efd64b433772c 2024-11-22T17:14:24,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/59fd11c2b9034f679e80a8f6960d57ad as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/59fd11c2b9034f679e80a8f6960d57ad 2024-11-22T17:14:24,951 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/59fd11c2b9034f679e80a8f6960d57ad, entries=150, sequenceid=92, filesize=11.7 K 2024-11-22T17:14:24,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/4a1cf233faad4279a5905af44a32a471 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/4a1cf233faad4279a5905af44a32a471 2024-11-22T17:14:24,968 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/4a1cf233faad4279a5905af44a32a471, entries=150, sequenceid=92, filesize=11.7 K 2024-11-22T17:14:24,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/d77ae0c438664a5da80efd64b433772c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/d77ae0c438664a5da80efd64b433772c 2024-11-22T17:14:24,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295724964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295724967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:24,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295724969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295724969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:24,985 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/d77ae0c438664a5da80efd64b433772c, entries=150, sequenceid=92, filesize=11.7 K 2024-11-22T17:14:24,987 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=154.31 KB/158010 for c70a0885ec0e4f41398ae6890c8dabe5 in 619ms, sequenceid=92, compaction requested=true 2024-11-22T17:14:24,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:24,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:24,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-22T17:14:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-22T17:14:24,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-22T17:14:24,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2480 sec 2024-11-22T17:14:24,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.2610 sec 2024-11-22T17:14:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:25,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=161.02 KB heapSize=447.23 KB 2024-11-22T17:14:25,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:25,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:25,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:25,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:25,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:25,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:25,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/98eb3d7fea0f4015b4960cbbe918f335 is 50, key is test_row_0/A:col10/1732295665023/Put/seqid=0 2024-11-22T17:14:25,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295725052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741855_1031 (size=14541) 2024-11-22T17:14:25,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/98eb3d7fea0f4015b4960cbbe918f335 2024-11-22T17:14:25,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295725082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295725082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/100c6aaffcdd4a39ae3008f2bf44618d is 50, key is test_row_0/B:col10/1732295665023/Put/seqid=0 2024-11-22T17:14:25,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741856_1032 (size=12151) 2024-11-22T17:14:25,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295725092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295725092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/100c6aaffcdd4a39ae3008f2bf44618d 2024-11-22T17:14:25,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/fcc23f9bf66c46fd97e4f14a1186bcb1 is 50, key is test_row_0/C:col10/1732295665023/Put/seqid=0 2024-11-22T17:14:25,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741857_1033 (size=12151) 2024-11-22T17:14:25,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/fcc23f9bf66c46fd97e4f14a1186bcb1 2024-11-22T17:14:25,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295725162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/98eb3d7fea0f4015b4960cbbe918f335 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/98eb3d7fea0f4015b4960cbbe918f335 2024-11-22T17:14:25,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/98eb3d7fea0f4015b4960cbbe918f335, entries=200, sequenceid=198, filesize=14.2 K 2024-11-22T17:14:25,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/100c6aaffcdd4a39ae3008f2bf44618d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/100c6aaffcdd4a39ae3008f2bf44618d 2024-11-22T17:14:25,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/100c6aaffcdd4a39ae3008f2bf44618d, entries=150, sequenceid=198, filesize=11.9 K 2024-11-22T17:14:25,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/fcc23f9bf66c46fd97e4f14a1186bcb1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/fcc23f9bf66c46fd97e4f14a1186bcb1 2024-11-22T17:14:25,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/fcc23f9bf66c46fd97e4f14a1186bcb1, entries=150, sequenceid=198, filesize=11.9 K 2024-11-22T17:14:25,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~447.19 KB/457920, currentSize=26.84 KB/27480 for c70a0885ec0e4f41398ae6890c8dabe5 in 198ms, sequenceid=198, compaction requested=true 2024-11-22T17:14:25,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:25,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:14:25,224 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:25,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:25,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:14:25,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:25,225 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:25,232 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52987 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:25,232 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/A is initiating minor compaction (all files) 2024-11-22T17:14:25,233 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/A in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:25,233 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:25,233 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/B is initiating minor compaction (all files) 2024-11-22T17:14:25,233 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/c6e5b27aa6014a998e3b31aa4d56e312, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/9594292a1792425bbe77baf95b9756f1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/59fd11c2b9034f679e80a8f6960d57ad, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/98eb3d7fea0f4015b4960cbbe918f335] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=51.7 K 2024-11-22T17:14:25,233 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/B in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:25,233 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/82680d69cf6e43afa382cfa67fb7fc98, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/7eaf27ef7f1548e583e2a3a2dc6dbc41, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/4a1cf233faad4279a5905af44a32a471, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/100c6aaffcdd4a39ae3008f2bf44618d] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=47.1 K 2024-11-22T17:14:25,235 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 82680d69cf6e43afa382cfa67fb7fc98, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732295663000 2024-11-22T17:14:25,235 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6e5b27aa6014a998e3b31aa4d56e312, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732295663000 2024-11-22T17:14:25,236 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eaf27ef7f1548e583e2a3a2dc6dbc41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732295663154 2024-11-22T17:14:25,236 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9594292a1792425bbe77baf95b9756f1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732295663154 2024-11-22T17:14:25,238 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a1cf233faad4279a5905af44a32a471, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732295663313 2024-11-22T17:14:25,238 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59fd11c2b9034f679e80a8f6960d57ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732295663313 2024-11-22T17:14:25,239 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98eb3d7fea0f4015b4960cbbe918f335, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732295664959 2024-11-22T17:14:25,239 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 100c6aaffcdd4a39ae3008f2bf44618d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732295664959 2024-11-22T17:14:25,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:14:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:25,264 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#A#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:25,264 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/d24f3821a7864876b4c1051131cace27 is 50, key is test_row_0/A:col10/1732295665023/Put/seqid=0 2024-11-22T17:14:25,278 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#B#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:25,279 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/3cb686fab3764eefb04e90dd146b680b is 50, key is test_row_0/B:col10/1732295665023/Put/seqid=0 2024-11-22T17:14:25,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741859_1035 (size=12391) 2024-11-22T17:14:25,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741858_1034 (size=12391) 2024-11-22T17:14:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:25,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T17:14:25,309 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/d24f3821a7864876b4c1051131cace27 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/d24f3821a7864876b4c1051131cace27 2024-11-22T17:14:25,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:25,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:25,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:25,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:25,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:25,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:25,321 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/3cb686fab3764eefb04e90dd146b680b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/3cb686fab3764eefb04e90dd146b680b 2024-11-22T17:14:25,326 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/A of c70a0885ec0e4f41398ae6890c8dabe5 into d24f3821a7864876b4c1051131cace27(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:25,326 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:25,326 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/A, priority=12, startTime=1732295665224; duration=0sec 2024-11-22T17:14:25,327 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:25,327 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:A 2024-11-22T17:14:25,327 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:25,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/bfab417825734ce4bfd07758330d085b is 50, key is test_row_0/A:col10/1732295665306/Put/seqid=0 2024-11-22T17:14:25,334 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:25,335 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/C is initiating minor compaction (all files) 2024-11-22T17:14:25,335 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/C in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:25,335 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/c38071bb8116403c8e2c03b2c207085d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/91bf32e73c7f4e71b255e96b95d06956, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/d77ae0c438664a5da80efd64b433772c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/fcc23f9bf66c46fd97e4f14a1186bcb1] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=47.1 K 2024-11-22T17:14:25,337 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting c38071bb8116403c8e2c03b2c207085d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732295663000 2024-11-22T17:14:25,339 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91bf32e73c7f4e71b255e96b95d06956, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732295663154 2024-11-22T17:14:25,340 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting d77ae0c438664a5da80efd64b433772c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732295663313 2024-11-22T17:14:25,342 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcc23f9bf66c46fd97e4f14a1186bcb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732295664959 2024-11-22T17:14:25,346 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/B of c70a0885ec0e4f41398ae6890c8dabe5 into 3cb686fab3764eefb04e90dd146b680b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:25,346 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:25,346 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/B, priority=12, startTime=1732295665224; duration=0sec 2024-11-22T17:14:25,347 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:25,347 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:B 2024-11-22T17:14:25,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741860_1036 (size=12151) 2024-11-22T17:14:25,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/bfab417825734ce4bfd07758330d085b 2024-11-22T17:14:25,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/85554f4438ee44d2bc871f18772d6150 is 50, key is test_row_0/B:col10/1732295665306/Put/seqid=0 2024-11-22T17:14:25,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295725376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295725376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295725379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295725382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,393 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#C#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:25,394 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/763a24d0bdf04c8db4b5b801b6e1e6ad is 50, key is test_row_0/C:col10/1732295665023/Put/seqid=0 2024-11-22T17:14:25,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295725387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741861_1037 (size=12151) 2024-11-22T17:14:25,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/85554f4438ee44d2bc871f18772d6150 2024-11-22T17:14:25,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741862_1038 (size=12391) 2024-11-22T17:14:25,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/3758e469874a417eb1d795038afe3ef8 is 50, key is test_row_0/C:col10/1732295665306/Put/seqid=0 2024-11-22T17:14:25,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741863_1039 (size=12151) 2024-11-22T17:14:25,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/3758e469874a417eb1d795038afe3ef8 2024-11-22T17:14:25,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295725492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295725492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/bfab417825734ce4bfd07758330d085b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/bfab417825734ce4bfd07758330d085b 2024-11-22T17:14:25,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/bfab417825734ce4bfd07758330d085b, entries=150, sequenceid=210, filesize=11.9 K 2024-11-22T17:14:25,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/85554f4438ee44d2bc871f18772d6150 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/85554f4438ee44d2bc871f18772d6150 2024-11-22T17:14:25,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295725509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295725509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/85554f4438ee44d2bc871f18772d6150, entries=150, sequenceid=210, filesize=11.9 K 2024-11-22T17:14:25,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/3758e469874a417eb1d795038afe3ef8 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/3758e469874a417eb1d795038afe3ef8 2024-11-22T17:14:25,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295725520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/3758e469874a417eb1d795038afe3ef8, entries=150, sequenceid=210, filesize=11.9 K 2024-11-22T17:14:25,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c70a0885ec0e4f41398ae6890c8dabe5 in 229ms, sequenceid=210, compaction requested=false 2024-11-22T17:14:25,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:25,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295665375, max timestamp=1732295665708Num uniques 859; 2024-11-22T17:14:25,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295665375, max timestamp=1732295665708Num uniques 859; 2024-11-22T17:14:25,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:25,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:25,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295665375, max timestamp=1732295665708Num uniques 859; 2024-11-22T17:14:25,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:25,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-22T17:14:25,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:25,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:25,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:25,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:25,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:25,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:25,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/1f4791e1e4a447099016cd3f6c335822 is 50, key is test_row_0/A:col10/1732295665725/Put/seqid=0 2024-11-22T17:14:25,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741864_1040 (size=12151) 2024-11-22T17:14:25,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/1f4791e1e4a447099016cd3f6c335822 2024-11-22T17:14:25,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/5a2fdf432a34444584036ae512e1617c is 50, key is test_row_0/B:col10/1732295665725/Put/seqid=0 2024-11-22T17:14:25,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295665771, max timestamp=1732295665785Num uniques 859; 2024-11-22T17:14:25,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295665771, max timestamp=1732295665785Num uniques 859; 2024-11-22T17:14:25,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:25,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:25,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295665771, max timestamp=1732295665785Num uniques 859; 2024-11-22T17:14:25,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:25,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741865_1041 (size=12151) 2024-11-22T17:14:25,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/5a2fdf432a34444584036ae512e1617c 2024-11-22T17:14:25,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295725818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295725820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295725820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295725822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295725824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,858 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/763a24d0bdf04c8db4b5b801b6e1e6ad as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/763a24d0bdf04c8db4b5b801b6e1e6ad 2024-11-22T17:14:25,875 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/C of c70a0885ec0e4f41398ae6890c8dabe5 into 763a24d0bdf04c8db4b5b801b6e1e6ad(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:25,875 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:25,875 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/C, priority=12, startTime=1732295665240; duration=0sec 2024-11-22T17:14:25,875 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:25,875 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:C 2024-11-22T17:14:25,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/11842d79f0b74a56aefa5272a335baef is 50, key is test_row_0/C:col10/1732295665725/Put/seqid=0 2024-11-22T17:14:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T17:14:25,882 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:25,882 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:14:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-22T17:14:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T17:14:25,886 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:14:25,887 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:14:25,887 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:14:25,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741866_1042 (size=12151) 2024-11-22T17:14:25,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295725941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295725941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295725941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295725941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295725941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T17:14:26,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T17:14:26,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:26,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:26,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:26,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:26,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:26,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295726151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295726152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295726152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295726152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295726152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T17:14:26,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:26,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:26,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:26,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:26,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T17:14:26,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/11842d79f0b74a56aefa5272a335baef 2024-11-22T17:14:26,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/1f4791e1e4a447099016cd3f6c335822 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/1f4791e1e4a447099016cd3f6c335822 2024-11-22T17:14:26,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/1f4791e1e4a447099016cd3f6c335822, entries=150, sequenceid=238, filesize=11.9 K 2024-11-22T17:14:26,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/5a2fdf432a34444584036ae512e1617c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/5a2fdf432a34444584036ae512e1617c 2024-11-22T17:14:26,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/5a2fdf432a34444584036ae512e1617c, entries=150, sequenceid=238, filesize=11.9 K 2024-11-22T17:14:26,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/11842d79f0b74a56aefa5272a335baef as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/11842d79f0b74a56aefa5272a335baef 2024-11-22T17:14:26,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/11842d79f0b74a56aefa5272a335baef, entries=150, sequenceid=238, filesize=11.9 K 2024-11-22T17:14:26,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=127.47 KB/130530 for c70a0885ec0e4f41398ae6890c8dabe5 in 615ms, sequenceid=238, compaction requested=true 2024-11-22T17:14:26,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:26,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:14:26,345 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:14:26,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:26,345 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:14:26,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:14:26,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:26,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:14:26,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:26,346 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:14:26,347 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/A is initiating minor compaction (all files) 2024-11-22T17:14:26,347 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:14:26,347 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/A in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:26,347 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/B is initiating minor compaction (all files) 2024-11-22T17:14:26,347 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/B in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:26,347 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/d24f3821a7864876b4c1051131cace27, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/bfab417825734ce4bfd07758330d085b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/1f4791e1e4a447099016cd3f6c335822] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=35.8 K 2024-11-22T17:14:26,347 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/3cb686fab3764eefb04e90dd146b680b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/85554f4438ee44d2bc871f18772d6150, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/5a2fdf432a34444584036ae512e1617c] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=35.8 K 2024-11-22T17:14:26,348 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cb686fab3764eefb04e90dd146b680b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732295664959 2024-11-22T17:14:26,348 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting d24f3821a7864876b4c1051131cace27, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732295664959 2024-11-22T17:14:26,349 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 85554f4438ee44d2bc871f18772d6150, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732295665306 2024-11-22T17:14:26,349 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a2fdf432a34444584036ae512e1617c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732295665725 2024-11-22T17:14:26,349 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfab417825734ce4bfd07758330d085b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732295665306 2024-11-22T17:14:26,351 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f4791e1e4a447099016cd3f6c335822, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732295665725 2024-11-22T17:14:26,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T17:14:26,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:26,355 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-22T17:14:26,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:26,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:26,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:26,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:26,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:26,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:26,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/e634a0b22a57493abdd494a5eacf32ea is 50, key is test_row_0/A:col10/1732295665811/Put/seqid=0 2024-11-22T17:14:26,387 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:26,388 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/99487b642ea5469790a079479ef79df1 is 50, key is test_row_0/B:col10/1732295665725/Put/seqid=0 2024-11-22T17:14:26,390 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#A#compaction#32 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:26,391 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/57082167416a4ee0ae8164dde7e0f40f is 50, key is test_row_0/A:col10/1732295665725/Put/seqid=0 2024-11-22T17:14:26,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741867_1043 (size=12301) 2024-11-22T17:14:26,402 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/e634a0b22a57493abdd494a5eacf32ea 2024-11-22T17:14:26,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741868_1044 (size=12493) 2024-11-22T17:14:26,423 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/99487b642ea5469790a079479ef79df1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/99487b642ea5469790a079479ef79df1 2024-11-22T17:14:26,437 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/B of c70a0885ec0e4f41398ae6890c8dabe5 into 99487b642ea5469790a079479ef79df1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:26,437 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:26,437 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/B, priority=13, startTime=1732295666345; duration=0sec 2024-11-22T17:14:26,438 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:26,438 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:B 2024-11-22T17:14:26,438 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:14:26,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/7517bbf440a243f9aa54398167df8cb5 is 50, key is test_row_0/B:col10/1732295665811/Put/seqid=0 2024-11-22T17:14:26,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741869_1045 (size=12493) 2024-11-22T17:14:26,446 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:14:26,446 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/C is initiating minor compaction (all files) 2024-11-22T17:14:26,446 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/C in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:26,446 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/763a24d0bdf04c8db4b5b801b6e1e6ad, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/3758e469874a417eb1d795038afe3ef8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/11842d79f0b74a56aefa5272a335baef] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=35.8 K 2024-11-22T17:14:26,451 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 763a24d0bdf04c8db4b5b801b6e1e6ad, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732295664959 2024-11-22T17:14:26,454 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 3758e469874a417eb1d795038afe3ef8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732295665306 2024-11-22T17:14:26,455 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 11842d79f0b74a56aefa5272a335baef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732295665725 2024-11-22T17:14:26,457 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/57082167416a4ee0ae8164dde7e0f40f as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/57082167416a4ee0ae8164dde7e0f40f 2024-11-22T17:14:26,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741870_1046 (size=12301) 2024-11-22T17:14:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:26,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:26,477 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/A of c70a0885ec0e4f41398ae6890c8dabe5 into 57082167416a4ee0ae8164dde7e0f40f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:26,477 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:26,477 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/A, priority=13, startTime=1732295666344; duration=0sec 2024-11-22T17:14:26,477 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:26,477 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:A 2024-11-22T17:14:26,496 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#C#compaction#34 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:26,496 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/ec5d124683a8474993710892f7cc686e is 50, key is test_row_0/C:col10/1732295665725/Put/seqid=0 2024-11-22T17:14:26,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295726489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295726485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295726491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295726489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295726498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741871_1047 (size=12493) 2024-11-22T17:14:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T17:14:26,537 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/ec5d124683a8474993710892f7cc686e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/ec5d124683a8474993710892f7cc686e 2024-11-22T17:14:26,550 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/C of c70a0885ec0e4f41398ae6890c8dabe5 into ec5d124683a8474993710892f7cc686e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:26,550 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:26,550 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/C, priority=13, startTime=1732295666345; duration=0sec 2024-11-22T17:14:26,551 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:26,551 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:C 2024-11-22T17:14:26,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295726611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295726612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295726612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295726612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295726612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295726821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295726822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295726822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295726822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295726822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:26,869 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/7517bbf440a243f9aa54398167df8cb5 2024-11-22T17:14:26,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/e5e1e73e62fc4157bc723eb373b8675d is 50, key is test_row_0/C:col10/1732295665811/Put/seqid=0 2024-11-22T17:14:26,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741872_1048 (size=12301) 2024-11-22T17:14:26,913 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/e5e1e73e62fc4157bc723eb373b8675d 2024-11-22T17:14:26,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/e634a0b22a57493abdd494a5eacf32ea as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/e634a0b22a57493abdd494a5eacf32ea 2024-11-22T17:14:26,935 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/e634a0b22a57493abdd494a5eacf32ea, entries=150, sequenceid=274, filesize=12.0 K 2024-11-22T17:14:26,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/7517bbf440a243f9aa54398167df8cb5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/7517bbf440a243f9aa54398167df8cb5 2024-11-22T17:14:26,947 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/7517bbf440a243f9aa54398167df8cb5, entries=150, sequenceid=274, filesize=12.0 K 2024-11-22T17:14:26,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/e5e1e73e62fc4157bc723eb373b8675d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/e5e1e73e62fc4157bc723eb373b8675d 2024-11-22T17:14:26,962 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/e5e1e73e62fc4157bc723eb373b8675d, entries=150, sequenceid=274, filesize=12.0 K 2024-11-22T17:14:26,965 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for c70a0885ec0e4f41398ae6890c8dabe5 in 610ms, sequenceid=274, compaction requested=false 2024-11-22T17:14:26,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:26,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:26,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-22T17:14:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-22T17:14:26,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-22T17:14:26,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0810 sec 2024-11-22T17:14:26,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.0930 sec 2024-11-22T17:14:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T17:14:27,022 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:27,023 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:14:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-22T17:14:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T17:14:27,030 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:14:27,035 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:14:27,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:14:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T17:14:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:27,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T17:14:27,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:27,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:27,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:27,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:27,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:27,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:27,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/dda020ff1e274d38a5905aa630f48c49 is 50, key is test_row_0/A:col10/1732295666485/Put/seqid=0 2024-11-22T17:14:27,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295727163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295727165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295727167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295727169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741873_1049 (size=12301) 2024-11-22T17:14:27,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/dda020ff1e274d38a5905aa630f48c49 2024-11-22T17:14:27,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295727169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T17:14:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:27,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/39edb64c590340d48f8ba7b7dbf462a9 is 50, key is test_row_0/B:col10/1732295666485/Put/seqid=0 2024-11-22T17:14:27,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741874_1050 (size=12301) 2024-11-22T17:14:27,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/39edb64c590340d48f8ba7b7dbf462a9 2024-11-22T17:14:27,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/861e745b7ebc4fde8240a09f4a9a6aaf is 50, key is test_row_0/C:col10/1732295666485/Put/seqid=0 2024-11-22T17:14:27,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741875_1051 (size=12301) 2024-11-22T17:14:27,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295727272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295727281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295727281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295727282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295727292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T17:14:27,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T17:14:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295727482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295727491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295727491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295727492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T17:14:27,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:27,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295727507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T17:14:27,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T17:14:27,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:27,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/861e745b7ebc4fde8240a09f4a9a6aaf 2024-11-22T17:14:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/dda020ff1e274d38a5905aa630f48c49 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/dda020ff1e274d38a5905aa630f48c49 2024-11-22T17:14:27,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/dda020ff1e274d38a5905aa630f48c49, entries=150, sequenceid=290, filesize=12.0 K 2024-11-22T17:14:27,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/39edb64c590340d48f8ba7b7dbf462a9 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/39edb64c590340d48f8ba7b7dbf462a9 2024-11-22T17:14:27,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/39edb64c590340d48f8ba7b7dbf462a9, entries=150, sequenceid=290, filesize=12.0 K 2024-11-22T17:14:27,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/861e745b7ebc4fde8240a09f4a9a6aaf as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/861e745b7ebc4fde8240a09f4a9a6aaf 2024-11-22T17:14:27,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/861e745b7ebc4fde8240a09f4a9a6aaf, entries=150, sequenceid=290, filesize=12.0 K 2024-11-22T17:14:27,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c70a0885ec0e4f41398ae6890c8dabe5 in 571ms, sequenceid=290, compaction requested=true 2024-11-22T17:14:27,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:27,707 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:14:27,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:14:27,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:27,708 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:14:27,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:14:27,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:27,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:14:27,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:27,709 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:14:27,709 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/A is initiating minor compaction (all files) 2024-11-22T17:14:27,710 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/A in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,710 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/57082167416a4ee0ae8164dde7e0f40f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/e634a0b22a57493abdd494a5eacf32ea, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/dda020ff1e274d38a5905aa630f48c49] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=36.2 K 2024-11-22T17:14:27,710 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:14:27,710 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/B is initiating minor compaction (all files) 2024-11-22T17:14:27,710 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/B in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,710 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/99487b642ea5469790a079479ef79df1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/7517bbf440a243f9aa54398167df8cb5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/39edb64c590340d48f8ba7b7dbf462a9] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=36.2 K 2024-11-22T17:14:27,711 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57082167416a4ee0ae8164dde7e0f40f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732295665725 2024-11-22T17:14:27,711 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 99487b642ea5469790a079479ef79df1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732295665725 2024-11-22T17:14:27,711 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting e634a0b22a57493abdd494a5eacf32ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732295665811 2024-11-22T17:14:27,712 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 7517bbf440a243f9aa54398167df8cb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732295665811 2024-11-22T17:14:27,714 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting dda020ff1e274d38a5905aa630f48c49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732295666485 2024-11-22T17:14:27,714 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 39edb64c590340d48f8ba7b7dbf462a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732295666485 2024-11-22T17:14:27,730 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#A#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:27,731 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/af9a2b27b1e94ce29693161167af6aa9 is 50, key is test_row_0/A:col10/1732295666485/Put/seqid=0 2024-11-22T17:14:27,736 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:27,737 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/61c9148a7d754420986893b8989ebd37 is 50, key is test_row_0/B:col10/1732295666485/Put/seqid=0 2024-11-22T17:14:27,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741876_1052 (size=12745) 2024-11-22T17:14:27,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741877_1053 (size=12745) 2024-11-22T17:14:27,789 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/61c9148a7d754420986893b8989ebd37 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/61c9148a7d754420986893b8989ebd37 2024-11-22T17:14:27,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295667164, max timestamp=1732295667793Num uniques 859; 2024-11-22T17:14:27,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:27,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295667164, max timestamp=1732295667793Num uniques 859; 2024-11-22T17:14:27,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:27,803 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/B of c70a0885ec0e4f41398ae6890c8dabe5 into 61c9148a7d754420986893b8989ebd37(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:27,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-22T17:14:27,803 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:27,803 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/B, priority=13, startTime=1732295667708; duration=0sec 2024-11-22T17:14:27,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:27,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:27,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:27,804 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:27,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:27,804 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:B 2024-11-22T17:14:27,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:27,804 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:14:27,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:27,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T17:14:27,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:27,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,816 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:14:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,816 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/C is initiating minor compaction (all files) 2024-11-22T17:14:27,816 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/C in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,816 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/ec5d124683a8474993710892f7cc686e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/e5e1e73e62fc4157bc723eb373b8675d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/861e745b7ebc4fde8240a09f4a9a6aaf] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=36.2 K 2024-11-22T17:14:27,817 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting ec5d124683a8474993710892f7cc686e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732295665725 2024-11-22T17:14:27,818 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting e5e1e73e62fc4157bc723eb373b8675d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732295665811 2024-11-22T17:14:27,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/b9be9afff0094859adec22f044fec6aa is 50, key is test_row_0/A:col10/1732295667793/Put/seqid=0 2024-11-22T17:14:27,819 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 861e745b7ebc4fde8240a09f4a9a6aaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732295666485 2024-11-22T17:14:27,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741878_1054 (size=14741) 2024-11-22T17:14:27,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/b9be9afff0094859adec22f044fec6aa 2024-11-22T17:14:27,843 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:27,844 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/4282b33266ce4bcb8ebe5abd8b62b53b is 50, key is test_row_0/C:col10/1732295666485/Put/seqid=0 2024-11-22T17:14:27,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741879_1055 (size=12745) 2024-11-22T17:14:27,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/6850a7fc032c4f71bc93fc21ff4385dc is 50, key is test_row_0/B:col10/1732295667793/Put/seqid=0 2024-11-22T17:14:27,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295667842, max timestamp=1732295667862Num uniques 859; 2024-11-22T17:14:27,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:27,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295667842, max timestamp=1732295667862Num uniques 859; 2024-11-22T17:14:27,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295667842, max timestamp=1732295667862Num uniques 859; 2024-11-22T17:14:27,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:27,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:27,877 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/4282b33266ce4bcb8ebe5abd8b62b53b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/4282b33266ce4bcb8ebe5abd8b62b53b 2024-11-22T17:14:27,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741880_1056 (size=12301) 2024-11-22T17:14:27,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/6850a7fc032c4f71bc93fc21ff4385dc 2024-11-22T17:14:27,892 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/C of c70a0885ec0e4f41398ae6890c8dabe5 into 4282b33266ce4bcb8ebe5abd8b62b53b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:27,892 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:27,892 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/C, priority=13, startTime=1732295667709; duration=0sec 2024-11-22T17:14:27,892 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:27,892 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:C 2024-11-22T17:14:27,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/5f179a980a9947c78a847d8e960d502e is 50, key is test_row_0/C:col10/1732295667793/Put/seqid=0 2024-11-22T17:14:27,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295727900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295727902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741881_1057 (size=12301) 2024-11-22T17:14:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295727905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295727907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295727909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:27,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T17:14:27,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:27,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:27,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:28,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295728011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295728022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295728022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295728023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295728033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T17:14:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:28,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T17:14:28,194 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/af9a2b27b1e94ce29693161167af6aa9 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/af9a2b27b1e94ce29693161167af6aa9 2024-11-22T17:14:28,216 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/A of c70a0885ec0e4f41398ae6890c8dabe5 into af9a2b27b1e94ce29693161167af6aa9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:28,216 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:28,216 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/A, priority=13, startTime=1732295667707; duration=0sec 2024-11-22T17:14:28,217 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:28,217 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:A 2024-11-22T17:14:28,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295728229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295728237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295728237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295728242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295728242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T17:14:28,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:28,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:28,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:28,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:28,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:28,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/5f179a980a9947c78a847d8e960d502e 2024-11-22T17:14:28,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/b9be9afff0094859adec22f044fec6aa as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/b9be9afff0094859adec22f044fec6aa 2024-11-22T17:14:28,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/b9be9afff0094859adec22f044fec6aa, entries=200, sequenceid=312, filesize=14.4 K 2024-11-22T17:14:28,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/6850a7fc032c4f71bc93fc21ff4385dc as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6850a7fc032c4f71bc93fc21ff4385dc 2024-11-22T17:14:28,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6850a7fc032c4f71bc93fc21ff4385dc, entries=150, sequenceid=312, filesize=12.0 K 2024-11-22T17:14:28,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/5f179a980a9947c78a847d8e960d502e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5f179a980a9947c78a847d8e960d502e 2024-11-22T17:14:28,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5f179a980a9947c78a847d8e960d502e, entries=150, sequenceid=312, filesize=12.0 K 2024-11-22T17:14:28,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for c70a0885ec0e4f41398ae6890c8dabe5 in 550ms, sequenceid=312, compaction requested=false 2024-11-22T17:14:28,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:28,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T17:14:28,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:28,441 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-22T17:14:28,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:28,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:28,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:28,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:28,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:28,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:28,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/fb3a063ab5bf42909a6dc38b53245adb is 50, key is test_row_0/A:col10/1732295667902/Put/seqid=0 2024-11-22T17:14:28,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741882_1058 (size=12301) 2024-11-22T17:14:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:28,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:28,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295728566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295728568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295728571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295728574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295728582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295728681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295728682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295728682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295728691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295728701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,857 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/fb3a063ab5bf42909a6dc38b53245adb 2024-11-22T17:14:28,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/32a9b05031bc4539b1a1a7106f3acd5e is 50, key is test_row_0/B:col10/1732295667902/Put/seqid=0 2024-11-22T17:14:28,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741883_1059 (size=12301) 2024-11-22T17:14:28,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295728891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295728892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295728892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295728901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:28,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295728911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T17:14:29,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295729202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295729202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295729202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295729211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295729222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,288 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/32a9b05031bc4539b1a1a7106f3acd5e 2024-11-22T17:14:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/ff8a1b64b7204b2ca60a7bb0debc7ebc is 50, key is test_row_0/C:col10/1732295667902/Put/seqid=0 2024-11-22T17:14:29,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741884_1060 (size=12301) 2024-11-22T17:14:29,305 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/ff8a1b64b7204b2ca60a7bb0debc7ebc 2024-11-22T17:14:29,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/fb3a063ab5bf42909a6dc38b53245adb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/fb3a063ab5bf42909a6dc38b53245adb 2024-11-22T17:14:29,318 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/fb3a063ab5bf42909a6dc38b53245adb, entries=150, sequenceid=350, filesize=12.0 K 2024-11-22T17:14:29,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/32a9b05031bc4539b1a1a7106f3acd5e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/32a9b05031bc4539b1a1a7106f3acd5e 2024-11-22T17:14:29,330 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/32a9b05031bc4539b1a1a7106f3acd5e, entries=150, sequenceid=350, filesize=12.0 K 2024-11-22T17:14:29,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/ff8a1b64b7204b2ca60a7bb0debc7ebc as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/ff8a1b64b7204b2ca60a7bb0debc7ebc 2024-11-22T17:14:29,340 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/ff8a1b64b7204b2ca60a7bb0debc7ebc, entries=150, sequenceid=350, filesize=12.0 K 2024-11-22T17:14:29,342 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for c70a0885ec0e4f41398ae6890c8dabe5 in 901ms, sequenceid=350, compaction requested=true 2024-11-22T17:14:29,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:29,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:29,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-22T17:14:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-22T17:14:29,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-22T17:14:29,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3100 sec 2024-11-22T17:14:29,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.3260 sec 2024-11-22T17:14:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:29,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T17:14:29,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:29,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:29,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:29,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:29,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:29,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:29,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/24791c8083ba4c8db00d7a7a57f66c3c is 50, key is test_row_0/A:col10/1732295668569/Put/seqid=0 2024-11-22T17:14:29,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741885_1061 (size=14741) 2024-11-22T17:14:29,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295729747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295729743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295729749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295729751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295729753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295729861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295729862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295729862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295729862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:29,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295729862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:29,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:30,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295730071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:30,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:30,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295730071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:30,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:30,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295730081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:30,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:30,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295730081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295730081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:30,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/24791c8083ba4c8db00d7a7a57f66c3c 2024-11-22T17:14:30,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/07bebd19f1974b329e5586f46bfe1bbd is 50, key is test_row_0/B:col10/1732295668569/Put/seqid=0 2024-11-22T17:14:30,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741886_1062 (size=12301) 2024-11-22T17:14:30,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/07bebd19f1974b329e5586f46bfe1bbd 2024-11-22T17:14:30,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/d8c17da044e4463697ad4169c274051c is 50, key is test_row_0/C:col10/1732295668569/Put/seqid=0 2024-11-22T17:14:30,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741887_1063 (size=12301) 2024-11-22T17:14:30,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295730381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:30,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295730382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:30,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295730391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:30,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:30,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295730401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295730401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:30,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/d8c17da044e4463697ad4169c274051c 2024-11-22T17:14:30,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/24791c8083ba4c8db00d7a7a57f66c3c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/24791c8083ba4c8db00d7a7a57f66c3c 2024-11-22T17:14:30,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/24791c8083ba4c8db00d7a7a57f66c3c, entries=200, sequenceid=363, filesize=14.4 K 2024-11-22T17:14:30,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/07bebd19f1974b329e5586f46bfe1bbd as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/07bebd19f1974b329e5586f46bfe1bbd 2024-11-22T17:14:30,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/07bebd19f1974b329e5586f46bfe1bbd, entries=150, sequenceid=363, filesize=12.0 K 2024-11-22T17:14:30,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/d8c17da044e4463697ad4169c274051c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/d8c17da044e4463697ad4169c274051c 2024-11-22T17:14:30,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/d8c17da044e4463697ad4169c274051c, entries=150, sequenceid=363, filesize=12.0 K 2024-11-22T17:14:30,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c70a0885ec0e4f41398ae6890c8dabe5 in 903ms, sequenceid=363, compaction requested=true 2024-11-22T17:14:30,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:30,616 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:30,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:14:30,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:30,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:14:30,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:30,617 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:30,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:14:30,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:30,618 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54528 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:30,618 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/A is initiating minor compaction (all files) 2024-11-22T17:14:30,618 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/A in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:30,619 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/af9a2b27b1e94ce29693161167af6aa9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/b9be9afff0094859adec22f044fec6aa, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/fb3a063ab5bf42909a6dc38b53245adb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/24791c8083ba4c8db00d7a7a57f66c3c] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=53.3 K 2024-11-22T17:14:30,619 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting af9a2b27b1e94ce29693161167af6aa9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732295666485 2024-11-22T17:14:30,619 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:30,619 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/B is initiating minor compaction (all files) 2024-11-22T17:14:30,619 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/B in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:30,619 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/61c9148a7d754420986893b8989ebd37, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6850a7fc032c4f71bc93fc21ff4385dc, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/32a9b05031bc4539b1a1a7106f3acd5e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/07bebd19f1974b329e5586f46bfe1bbd] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=48.5 K 2024-11-22T17:14:30,621 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9be9afff0094859adec22f044fec6aa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732295667164 2024-11-22T17:14:30,622 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb3a063ab5bf42909a6dc38b53245adb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732295667902 2024-11-22T17:14:30,623 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24791c8083ba4c8db00d7a7a57f66c3c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732295668561 2024-11-22T17:14:30,623 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 61c9148a7d754420986893b8989ebd37, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732295666485 2024-11-22T17:14:30,624 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 6850a7fc032c4f71bc93fc21ff4385dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732295667166 2024-11-22T17:14:30,625 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 32a9b05031bc4539b1a1a7106f3acd5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732295667902 2024-11-22T17:14:30,625 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 07bebd19f1974b329e5586f46bfe1bbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732295668561 2024-11-22T17:14:30,637 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:30,637 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:30,638 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/8556d592606048c8b7feab676f217470 is 50, key is test_row_0/A:col10/1732295668569/Put/seqid=0 2024-11-22T17:14:30,638 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/5a50818973344beeacf67e28fb108d2d is 50, key is test_row_0/B:col10/1732295668569/Put/seqid=0 2024-11-22T17:14:30,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741888_1064 (size=12881) 2024-11-22T17:14:30,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741889_1065 (size=12881) 2024-11-22T17:14:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:30,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295669740, max timestamp=1732295670893Num uniques 859; 2024-11-22T17:14:30,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295669740, max timestamp=1732295670893Num uniques 859; 2024-11-22T17:14:30,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-22T17:14:30,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732295669740, max timestamp=1732295670893Num uniques 859; 2024-11-22T17:14:30,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:30,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:30,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:30,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:30,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:30,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:30,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:30,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:30,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:30,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/f05ebd86d8aa4ebf9bf8ab9c8c36819f is 50, key is test_row_0/A:col10/1732295670892/Put/seqid=0 2024-11-22T17:14:30,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741890_1066 (size=12301) 2024-11-22T17:14:30,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295670919, max timestamp=1732295670936Num uniques 859; 2024-11-22T17:14:30,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295670919, max timestamp=1732295670936Num uniques 859; 2024-11-22T17:14:30,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295670919, max timestamp=1732295670936Num uniques 859; 2024-11-22T17:14:30,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:30,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:30,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:30,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12264, totalHeapSize=28112, min timestamp=1732295670953, max timestamp=1732295670972Num uniques 1068; 2024-11-22T17:14:30,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48904, new segment data size=12264, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-22T17:14:30,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12261, totalHeapSize=28112, min timestamp=1732295670953, max timestamp=1732295670972Num uniques 1068; 2024-11-22T17:14:30,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48901, new segment data size=12261, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-22T17:14:30,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732295670953, max timestamp=1732295670972Num uniques 1068; 2024-11-22T17:14:30,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-22T17:14:31,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295730998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1732295670992, max timestamp=1732295671002Num uniques 1127; 2024-11-22T17:14:31,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1732295670992, max timestamp=1732295671002Num uniques 1127; 2024-11-22T17:14:31,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1732295670992, max timestamp=1732295671002Num uniques 1127; 2024-11-22T17:14:31,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=17262, suffix heap size=145176,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=377 2024-11-22T17:14:31,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=17262, suffix heap size=145176,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=377 2024-11-22T17:14:31,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=17262, suffix heap size=145176,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=377 2024-11-22T17:14:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295731044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732295671024, max timestamp=1732295671044Num uniques 1236; 2024-11-22T17:14:31,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=13096, suffix heap size=156512,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=286 2024-11-22T17:14:31,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732295671024, max timestamp=1732295671044Num uniques 1236; 2024-11-22T17:14:31,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=13096, suffix heap size=156512,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=286 2024-11-22T17:14:31,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732295671024, max timestamp=1732295671044Num uniques 1236; 2024-11-22T17:14:31,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=13096, suffix heap size=156512,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=286 2024-11-22T17:14:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,056 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/5a50818973344beeacf67e28fb108d2d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/5a50818973344beeacf67e28fb108d2d 2024-11-22T17:14:31,057 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/8556d592606048c8b7feab676f217470 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/8556d592606048c8b7feab676f217470 2024-11-22T17:14:31,067 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/B of c70a0885ec0e4f41398ae6890c8dabe5 into 5a50818973344beeacf67e28fb108d2d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:31,068 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:31,068 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/B, priority=12, startTime=1732295670617; duration=0sec 2024-11-22T17:14:31,068 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:31,068 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:B 2024-11-22T17:14:31,068 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:31,071 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:31,071 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/C is initiating minor compaction (all files) 2024-11-22T17:14:31,071 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/C in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:31,072 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/4282b33266ce4bcb8ebe5abd8b62b53b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5f179a980a9947c78a847d8e960d502e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/ff8a1b64b7204b2ca60a7bb0debc7ebc, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/d8c17da044e4463697ad4169c274051c] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=48.5 K 2024-11-22T17:14:31,072 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 4282b33266ce4bcb8ebe5abd8b62b53b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732295666485 2024-11-22T17:14:31,073 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f179a980a9947c78a847d8e960d502e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732295667166 2024-11-22T17:14:31,073 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting ff8a1b64b7204b2ca60a7bb0debc7ebc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732295667902 2024-11-22T17:14:31,074 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting d8c17da044e4463697ad4169c274051c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732295668561 2024-11-22T17:14:31,074 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/A of c70a0885ec0e4f41398ae6890c8dabe5 into 8556d592606048c8b7feab676f217470(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:31,074 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:31,074 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/A, priority=12, startTime=1732295670616; duration=0sec 2024-11-22T17:14:31,076 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:31,076 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:A 2024-11-22T17:14:31,087 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#C#compaction#54 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:31,088 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/2b14eb52abd648a0bbccf44149af5c3c is 50, key is test_row_0/C:col10/1732295668569/Put/seqid=0 2024-11-22T17:14:31,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741891_1067 (size=12881) 2024-11-22T17:14:31,105 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/2b14eb52abd648a0bbccf44149af5c3c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/2b14eb52abd648a0bbccf44149af5c3c 2024-11-22T17:14:31,131 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/C of c70a0885ec0e4f41398ae6890c8dabe5 into 2b14eb52abd648a0bbccf44149af5c3c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:31,131 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:31,131 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/C, priority=12, startTime=1732295670617; duration=0sec 2024-11-22T17:14:31,131 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:31,132 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:C 2024-11-22T17:14:31,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295731132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732295671089, max timestamp=1732295671136Num uniques 1145; 2024-11-22T17:14:31,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-22T17:14:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295731132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732295671089, max timestamp=1732295671136Num uniques 1145; 2024-11-22T17:14:31,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-22T17:14:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732295671089, max timestamp=1732295671136Num uniques 1145; 2024-11-22T17:14:31,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-22T17:14:31,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732295671159, max timestamp=1732295671180Num uniques 1054; 2024-11-22T17:14:31,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732295671159, max timestamp=1732295671180Num uniques 1054; 2024-11-22T17:14:31,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732295671159, max timestamp=1732295671180Num uniques 1054; 2024-11-22T17:14:31,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=13920, suffix heap size=137584,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=304 2024-11-22T17:14:31,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=13920, suffix heap size=137584,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=304 2024-11-22T17:14:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295731180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=13920, suffix heap size=137584,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=304 2024-11-22T17:14:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T17:14:31,192 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:31,192 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:14:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-22T17:14:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T17:14:31,196 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:14:31,197 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:14:31,197 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:14:31,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295731218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295731221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295731252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295731252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295731291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T17:14:31,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/f05ebd86d8aa4ebf9bf8ab9c8c36819f 2024-11-22T17:14:31,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/0486b51b3db3488fa567830567c3370b is 50, key is test_row_0/B:col10/1732295670892/Put/seqid=0 2024-11-22T17:14:31,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295731333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295731333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741892_1068 (size=12301) 2024-11-22T17:14:31,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/0486b51b3db3488fa567830567c3370b 2024-11-22T17:14:31,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T17:14:31,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:31,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:31,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:31,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:31,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:31,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/605d9660f4a94d11ae2ddd2e7b41dcd8 is 50, key is test_row_0/C:col10/1732295670892/Put/seqid=0 2024-11-22T17:14:31,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741893_1069 (size=12301) 2024-11-22T17:14:31,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295731461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295731461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T17:14:31,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:31,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:31,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:31,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:31,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:31,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295731501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T17:14:31,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295731541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295731541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T17:14:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:31,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:31,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/605d9660f4a94d11ae2ddd2e7b41dcd8 2024-11-22T17:14:31,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295731771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295731771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/f05ebd86d8aa4ebf9bf8ab9c8c36819f as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/f05ebd86d8aa4ebf9bf8ab9c8c36819f 2024-11-22T17:14:31,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/f05ebd86d8aa4ebf9bf8ab9c8c36819f, entries=150, sequenceid=384, filesize=12.0 K 2024-11-22T17:14:31,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/0486b51b3db3488fa567830567c3370b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/0486b51b3db3488fa567830567c3370b 2024-11-22T17:14:31,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/0486b51b3db3488fa567830567c3370b, entries=150, sequenceid=384, filesize=12.0 K 2024-11-22T17:14:31,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/605d9660f4a94d11ae2ddd2e7b41dcd8 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/605d9660f4a94d11ae2ddd2e7b41dcd8 2024-11-22T17:14:31,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/605d9660f4a94d11ae2ddd2e7b41dcd8, entries=150, sequenceid=384, filesize=12.0 K 2024-11-22T17:14:31,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=154.31 KB/158010 for c70a0885ec0e4f41398ae6890c8dabe5 in 901ms, sequenceid=384, compaction requested=false 2024-11-22T17:14:31,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:31,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T17:14:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:31,812 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=154.31 KB heapSize=431.16 KB 2024-11-22T17:14:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:31,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:31,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:31,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T17:14:31,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:31,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T17:14:31,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:31,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T17:14:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T17:14:31,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/eb4f767eda514da28534012f0adf81b2 is 50, key is test_row_0/A:col10/1732295671217/Put/seqid=0 2024-11-22T17:14:31,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741894_1070 (size=14741) 2024-11-22T17:14:31,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295731834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295731851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295731852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:31,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:31,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295731941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:31,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:31,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:32,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295732151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:32,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:32,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:32,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:32,229 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/eb4f767eda514da28534012f0adf81b2 2024-11-22T17:14:32,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/a1883077a6544c60a0b50c113ad3cecb is 50, key is test_row_0/B:col10/1732295671217/Put/seqid=0 2024-11-22T17:14:32,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741895_1071 (size=12301) 2024-11-22T17:14:32,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295732281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:32,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295732281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T17:14:32,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295732361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:32,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295732361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:32,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295732461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:32,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:32,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:32,642 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/a1883077a6544c60a0b50c113ad3cecb 2024-11-22T17:14:32,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/df9b23c9d22947adaf5d65b4113b8fc3 is 50, key is test_row_0/C:col10/1732295671217/Put/seqid=0 2024-11-22T17:14:32,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741896_1072 (size=12301) 2024-11-22T17:14:32,661 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/df9b23c9d22947adaf5d65b4113b8fc3 2024-11-22T17:14:32,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/eb4f767eda514da28534012f0adf81b2 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/eb4f767eda514da28534012f0adf81b2 2024-11-22T17:14:32,675 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/eb4f767eda514da28534012f0adf81b2, entries=200, sequenceid=511, filesize=14.4 K 2024-11-22T17:14:32,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/a1883077a6544c60a0b50c113ad3cecb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/a1883077a6544c60a0b50c113ad3cecb 2024-11-22T17:14:32,686 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/a1883077a6544c60a0b50c113ad3cecb, entries=150, sequenceid=511, filesize=12.0 K 2024-11-22T17:14:32,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/df9b23c9d22947adaf5d65b4113b8fc3 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/df9b23c9d22947adaf5d65b4113b8fc3 2024-11-22T17:14:32,694 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/df9b23c9d22947adaf5d65b4113b8fc3, entries=150, sequenceid=511, filesize=12.0 K 2024-11-22T17:14:32,695 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~451.73 KB/462576, currentSize=20.13 KB/20610 for c70a0885ec0e4f41398ae6890c8dabe5 in 883ms, sequenceid=511, compaction requested=true 2024-11-22T17:14:32,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:32,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:32,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-22T17:14:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-22T17:14:32,698 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-22T17:14:32,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5000 sec 2024-11-22T17:14:32,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5070 sec 2024-11-22T17:14:32,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:14:32,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:32,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:32,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:32,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:32,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:32,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:32,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/81fed2f9cc504bdc82499d7e50c41014 is 50, key is test_row_0/A:col10/1732295672977/Put/seqid=0 2024-11-22T17:14:32,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741897_1073 (size=12301) 2024-11-22T17:14:33,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295673051, max timestamp=1732295673063Num uniques 859; 2024-11-22T17:14:33,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295673051, max timestamp=1732295673063Num uniques 859; 2024-11-22T17:14:33,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:33,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:33,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295673051, max timestamp=1732295673063Num uniques 859; 2024-11-22T17:14:33,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:33,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295733146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:33,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:33,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295733251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:33,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295733292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:33,296 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in c70a0885ec0e4f41398ae6890c8dabe5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:14:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:33,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:33,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295733301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:33,307 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in c70a0885ec0e4f41398ae6890c8dabe5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:14:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:33,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T17:14:33,342 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:33,342 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:14:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-22T17:14:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T17:14:33,344 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:14:33,344 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:14:33,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:14:33,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295733381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:33,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295733381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:33,389 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in c70a0885ec0e4f41398ae6890c8dabe5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:14:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:33,389 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in c70a0885ec0e4f41398ae6890c8dabe5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:14:33,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:33,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/81fed2f9cc504bdc82499d7e50c41014 2024-11-22T17:14:33,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/b11c5e8994a3433d8dce0bef46a8c869 is 50, key is test_row_0/B:col10/1732295672977/Put/seqid=0 2024-11-22T17:14:33,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741898_1074 (size=12301) 2024-11-22T17:14:33,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T17:14:33,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295733461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:33,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T17:14:33,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:33,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:33,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:33,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:33,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:33,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T17:14:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:33,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:33,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T17:14:33,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295733771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:33,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T17:14:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:33,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:33,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/b11c5e8994a3433d8dce0bef46a8c869 2024-11-22T17:14:33,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/b72c9683f9d24124bf57417a05699567 is 50, key is test_row_0/C:col10/1732295672977/Put/seqid=0 2024-11-22T17:14:33,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741899_1075 (size=12301) 2024-11-22T17:14:33,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T17:14:33,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:33,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:33,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:33,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:33,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:33,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T17:14:34,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T17:14:34,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/b72c9683f9d24124bf57417a05699567 2024-11-22T17:14:34,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/81fed2f9cc504bdc82499d7e50c41014 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/81fed2f9cc504bdc82499d7e50c41014 2024-11-22T17:14:34,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/81fed2f9cc504bdc82499d7e50c41014, entries=150, sequenceid=521, filesize=12.0 K 2024-11-22T17:14:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/b11c5e8994a3433d8dce0bef46a8c869 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/b11c5e8994a3433d8dce0bef46a8c869 2024-11-22T17:14:34,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/b11c5e8994a3433d8dce0bef46a8c869, entries=150, sequenceid=521, filesize=12.0 K 2024-11-22T17:14:34,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/b72c9683f9d24124bf57417a05699567 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/b72c9683f9d24124bf57417a05699567 2024-11-22T17:14:34,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/b72c9683f9d24124bf57417a05699567, entries=150, sequenceid=521, filesize=12.0 K 2024-11-22T17:14:34,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T17:14:34,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:34,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c70a0885ec0e4f41398ae6890c8dabe5 in 1283ms, sequenceid=521, compaction requested=true 2024-11-22T17:14:34,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:34,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:14:34,265 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:34,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:34,266 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:34,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:14:34,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:34,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:14:34,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:34,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,267 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52224 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:34,267 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/A is initiating minor compaction (all files) 2024-11-22T17:14:34,267 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/A in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,267 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/8556d592606048c8b7feab676f217470, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/f05ebd86d8aa4ebf9bf8ab9c8c36819f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/eb4f767eda514da28534012f0adf81b2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/81fed2f9cc504bdc82499d7e50c41014] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=51 K 2024-11-22T17:14:34,268 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:34,268 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8556d592606048c8b7feab676f217470, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732295668561 2024-11-22T17:14:34,268 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/B is initiating minor compaction (all files) 2024-11-22T17:14:34,268 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/B in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,268 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/5a50818973344beeacf67e28fb108d2d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/0486b51b3db3488fa567830567c3370b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/a1883077a6544c60a0b50c113ad3cecb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/b11c5e8994a3433d8dce0bef46a8c869] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=48.6 K 2024-11-22T17:14:34,268 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting f05ebd86d8aa4ebf9bf8ab9c8c36819f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732295669750 2024-11-22T17:14:34,269 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a50818973344beeacf67e28fb108d2d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732295668561 2024-11-22T17:14:34,269 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb4f767eda514da28534012f0adf81b2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1732295671209 2024-11-22T17:14:34,269 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 0486b51b3db3488fa567830567c3370b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732295669750 2024-11-22T17:14:34,269 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81fed2f9cc504bdc82499d7e50c41014, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732295672974 2024-11-22T17:14:34,270 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a1883077a6544c60a0b50c113ad3cecb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1732295671209 2024-11-22T17:14:34,270 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting b11c5e8994a3433d8dce0bef46a8c869, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732295672974 2024-11-22T17:14:34,280 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#A#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:34,281 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/95cc26b4455f4c77acf998f0ceaa3e49 is 50, key is test_row_0/A:col10/1732295672977/Put/seqid=0 2024-11-22T17:14:34,282 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:34,283 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/995dd5b00f1349d7aa721158d9588fde is 50, key is test_row_0/B:col10/1732295672977/Put/seqid=0 2024-11-22T17:14:34,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:34,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-22T17:14:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T17:14:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T17:14:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T17:14:34,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741900_1076 (size=13017) 2024-11-22T17:14:34,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/eed8274a39804df1bbc6a6cc5db32a46 is 50, key is test_row_0/A:col10/1732295673132/Put/seqid=0 2024-11-22T17:14:34,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741901_1077 (size=13017) 2024-11-22T17:14:34,308 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/95cc26b4455f4c77acf998f0ceaa3e49 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/95cc26b4455f4c77acf998f0ceaa3e49 2024-11-22T17:14:34,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741902_1078 (size=14741) 2024-11-22T17:14:34,317 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/A of c70a0885ec0e4f41398ae6890c8dabe5 into 95cc26b4455f4c77acf998f0ceaa3e49(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:34,317 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:34,317 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/A, priority=12, startTime=1732295674265; duration=0sec 2024-11-22T17:14:34,317 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:34,317 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:A 2024-11-22T17:14:34,317 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:34,319 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:34,319 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/C is initiating minor compaction (all files) 2024-11-22T17:14:34,319 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/C in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,319 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/2b14eb52abd648a0bbccf44149af5c3c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/605d9660f4a94d11ae2ddd2e7b41dcd8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/df9b23c9d22947adaf5d65b4113b8fc3, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/b72c9683f9d24124bf57417a05699567] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=48.6 K 2024-11-22T17:14:34,320 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b14eb52abd648a0bbccf44149af5c3c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732295668561 2024-11-22T17:14:34,320 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 605d9660f4a94d11ae2ddd2e7b41dcd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732295669750 2024-11-22T17:14:34,320 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting df9b23c9d22947adaf5d65b4113b8fc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1732295671209 2024-11-22T17:14:34,321 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting b72c9683f9d24124bf57417a05699567, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732295672974 2024-11-22T17:14:34,332 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#C#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:34,332 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/43828007be9d4df495c7879a55ebac49 is 50, key is test_row_0/C:col10/1732295672977/Put/seqid=0 2024-11-22T17:14:34,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741903_1079 (size=13017) 2024-11-22T17:14:34,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295734326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:34,346 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/43828007be9d4df495c7879a55ebac49 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/43828007be9d4df495c7879a55ebac49 2024-11-22T17:14:34,353 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/C of c70a0885ec0e4f41398ae6890c8dabe5 into 43828007be9d4df495c7879a55ebac49(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:34,353 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:34,353 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/C, priority=12, startTime=1732295674266; duration=0sec 2024-11-22T17:14:34,353 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:34,353 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:C 2024-11-22T17:14:34,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T17:14:34,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:34,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295734451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T17:14:34,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T17:14:34,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:34,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295734661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:34,696 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/995dd5b00f1349d7aa721158d9588fde as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/995dd5b00f1349d7aa721158d9588fde 2024-11-22T17:14:34,701 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/B of c70a0885ec0e4f41398ae6890c8dabe5 into 995dd5b00f1349d7aa721158d9588fde(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:34,702 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:34,702 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/B, priority=12, startTime=1732295674265; duration=0sec 2024-11-22T17:14:34,702 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:34,702 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:B 2024-11-22T17:14:34,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/eed8274a39804df1bbc6a6cc5db32a46 2024-11-22T17:14:34,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/4d8d52c9402344169af6751fddb1ad2a is 50, key is test_row_0/B:col10/1732295673132/Put/seqid=0 2024-11-22T17:14:34,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741904_1080 (size=12301) 2024-11-22T17:14:34,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T17:14:34,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:34,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T17:14:34,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:34,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:34,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:34,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295734971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:35,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T17:14:35,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:35,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:35,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:35,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:35,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:35,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/4d8d52c9402344169af6751fddb1ad2a 2024-11-22T17:14:35,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/6908076924e7454aaa3c1826285e745b is 50, key is test_row_0/C:col10/1732295673132/Put/seqid=0 2024-11-22T17:14:35,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741905_1081 (size=12301) 2024-11-22T17:14:35,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/6908076924e7454aaa3c1826285e745b 2024-11-22T17:14:35,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/eed8274a39804df1bbc6a6cc5db32a46 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/eed8274a39804df1bbc6a6cc5db32a46 2024-11-22T17:14:35,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/eed8274a39804df1bbc6a6cc5db32a46, entries=200, sequenceid=559, filesize=14.4 K 2024-11-22T17:14:35,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/4d8d52c9402344169af6751fddb1ad2a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/4d8d52c9402344169af6751fddb1ad2a 2024-11-22T17:14:35,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/4d8d52c9402344169af6751fddb1ad2a, entries=150, sequenceid=559, filesize=12.0 K 2024-11-22T17:14:35,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/6908076924e7454aaa3c1826285e745b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/6908076924e7454aaa3c1826285e745b 2024-11-22T17:14:35,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/6908076924e7454aaa3c1826285e745b, entries=150, sequenceid=559, filesize=12.0 K 2024-11-22T17:14:35,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for c70a0885ec0e4f41398ae6890c8dabe5 in 896ms, sequenceid=559, compaction requested=false 2024-11-22T17:14:35,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:35,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T17:14:35,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:35,186 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-22T17:14:35,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:35,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:35,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:35,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:35,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:35,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:35,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/a913c931cfe54b3e87b13fab800b14c0 is 50, key is test_row_0/A:col10/1732295674311/Put/seqid=0 2024-11-22T17:14:35,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741906_1082 (size=12301) 2024-11-22T17:14:35,204 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/a913c931cfe54b3e87b13fab800b14c0 2024-11-22T17:14:35,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/af6db8aea548404c83343a5d2f533f55 is 50, key is test_row_0/B:col10/1732295674311/Put/seqid=0 2024-11-22T17:14:35,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741907_1083 (size=12301) 2024-11-22T17:14:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:35,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:35,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295675324, max timestamp=1732295675350Num uniques 859; 2024-11-22T17:14:35,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:35,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295675324, max timestamp=1732295675350Num uniques 859; 2024-11-22T17:14:35,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:35,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295675324, max timestamp=1732295675350Num uniques 859; 2024-11-22T17:14:35,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:35,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295735395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:35,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:35,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295735395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:35,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295735401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:35,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295735401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:35,408 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in c70a0885ec0e4f41398ae6890c8dabe5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:14:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,408 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in c70a0885ec0e4f41398ae6890c8dabe5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:14:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:35,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295735481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T17:14:35,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295735501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295735502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:35,628 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/af6db8aea548404c83343a5d2f533f55 2024-11-22T17:14:35,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/2b40ecf216234f32866904ee3309923f is 50, key is test_row_0/C:col10/1732295674311/Put/seqid=0 2024-11-22T17:14:35,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741908_1084 (size=12301) 2024-11-22T17:14:35,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:35,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295735711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295735711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:36,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295736022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295736022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:36,042 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/2b40ecf216234f32866904ee3309923f 2024-11-22T17:14:36,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/a913c931cfe54b3e87b13fab800b14c0 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/a913c931cfe54b3e87b13fab800b14c0 2024-11-22T17:14:36,054 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/a913c931cfe54b3e87b13fab800b14c0, entries=150, sequenceid=571, filesize=12.0 K 2024-11-22T17:14:36,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/af6db8aea548404c83343a5d2f533f55 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/af6db8aea548404c83343a5d2f533f55 2024-11-22T17:14:36,060 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/af6db8aea548404c83343a5d2f533f55, entries=150, sequenceid=571, filesize=12.0 K 2024-11-22T17:14:36,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/2b40ecf216234f32866904ee3309923f as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/2b40ecf216234f32866904ee3309923f 2024-11-22T17:14:36,069 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/2b40ecf216234f32866904ee3309923f, entries=150, sequenceid=571, filesize=12.0 K 2024-11-22T17:14:36,070 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for c70a0885ec0e4f41398ae6890c8dabe5 in 884ms, sequenceid=571, compaction requested=true 2024-11-22T17:14:36,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:36,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:36,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-22T17:14:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-22T17:14:36,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-22T17:14:36,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7260 sec 2024-11-22T17:14:36,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.7320 sec 2024-11-22T17:14:36,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-22T17:14:36,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:36,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T17:14:36,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:36,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T17:14:36,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:36,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T17:14:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:36,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/ab22b9d68a83471c8d42556e82de74e8 is 50, key is test_row_0/A:col10/1732295675384/Put/seqid=0 2024-11-22T17:14:36,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741909_1085 (size=14741) 2024-11-22T17:14:36,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295736534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:36,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295736541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295736541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:36,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295736652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:36,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295736861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:36,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:36,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:36,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/ab22b9d68a83471c8d42556e82de74e8 2024-11-22T17:14:36,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/13200f7d08e2476fb9da91ec9b578d4e is 50, key is test_row_0/B:col10/1732295675384/Put/seqid=0 2024-11-22T17:14:36,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741910_1086 (size=12301) 2024-11-22T17:14:36,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/13200f7d08e2476fb9da91ec9b578d4e 2024-11-22T17:14:36,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/cf85914265dc441cae2b97794d438acd is 50, key is test_row_0/C:col10/1732295675384/Put/seqid=0 2024-11-22T17:14:36,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741911_1087 (size=12301) 2024-11-22T17:14:36,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/cf85914265dc441cae2b97794d438acd 2024-11-22T17:14:36,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/ab22b9d68a83471c8d42556e82de74e8 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/ab22b9d68a83471c8d42556e82de74e8 2024-11-22T17:14:36,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/ab22b9d68a83471c8d42556e82de74e8, entries=200, sequenceid=609, filesize=14.4 K 2024-11-22T17:14:36,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/13200f7d08e2476fb9da91ec9b578d4e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/13200f7d08e2476fb9da91ec9b578d4e 2024-11-22T17:14:36,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/13200f7d08e2476fb9da91ec9b578d4e, entries=150, sequenceid=609, filesize=12.0 K 2024-11-22T17:14:36,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/cf85914265dc441cae2b97794d438acd as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/cf85914265dc441cae2b97794d438acd 2024-11-22T17:14:36,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/cf85914265dc441cae2b97794d438acd, entries=150, sequenceid=609, filesize=12.0 K 2024-11-22T17:14:36,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for c70a0885ec0e4f41398ae6890c8dabe5 in 487ms, sequenceid=609, compaction requested=true 2024-11-22T17:14:36,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:36,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:14:36,988 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:36,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:36,988 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:36,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:14:36,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:36,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:14:36,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:36,991 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:36,991 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/B is initiating minor compaction (all files) 2024-11-22T17:14:36,991 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/B in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:36,991 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/995dd5b00f1349d7aa721158d9588fde, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/4d8d52c9402344169af6751fddb1ad2a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/af6db8aea548404c83343a5d2f533f55, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/13200f7d08e2476fb9da91ec9b578d4e] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=48.8 K 2024-11-22T17:14:36,992 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:36,992 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/A is initiating minor compaction (all files) 2024-11-22T17:14:36,992 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/A in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:36,992 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/95cc26b4455f4c77acf998f0ceaa3e49, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/eed8274a39804df1bbc6a6cc5db32a46, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/a913c931cfe54b3e87b13fab800b14c0, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/ab22b9d68a83471c8d42556e82de74e8] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=53.5 K 2024-11-22T17:14:36,995 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 995dd5b00f1349d7aa721158d9588fde, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732295672974 2024-11-22T17:14:36,995 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95cc26b4455f4c77acf998f0ceaa3e49, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732295672974 2024-11-22T17:14:36,995 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d8d52c9402344169af6751fddb1ad2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1732295673124 2024-11-22T17:14:36,996 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting eed8274a39804df1bbc6a6cc5db32a46, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1732295673124 2024-11-22T17:14:36,996 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting af6db8aea548404c83343a5d2f533f55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732295674311 2024-11-22T17:14:36,997 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting a913c931cfe54b3e87b13fab800b14c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732295674311 2024-11-22T17:14:36,997 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 13200f7d08e2476fb9da91ec9b578d4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1732295675384 2024-11-22T17:14:36,998 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab22b9d68a83471c8d42556e82de74e8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1732295675384 2024-11-22T17:14:37,019 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:37,019 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/8f33d475fa094832b6bf6cf30cc5f47b is 50, key is test_row_0/A:col10/1732295675384/Put/seqid=0 2024-11-22T17:14:37,023 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:37,023 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/6866b61d94de48f7a8e835cf2bc5a9ce is 50, key is test_row_0/B:col10/1732295675384/Put/seqid=0 2024-11-22T17:14:37,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741912_1088 (size=13153) 2024-11-22T17:14:37,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741913_1089 (size=13153) 2024-11-22T17:14:37,048 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/8f33d475fa094832b6bf6cf30cc5f47b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/8f33d475fa094832b6bf6cf30cc5f47b 2024-11-22T17:14:37,050 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/6866b61d94de48f7a8e835cf2bc5a9ce as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6866b61d94de48f7a8e835cf2bc5a9ce 2024-11-22T17:14:37,061 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/A of c70a0885ec0e4f41398ae6890c8dabe5 into 8f33d475fa094832b6bf6cf30cc5f47b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:37,061 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/B of c70a0885ec0e4f41398ae6890c8dabe5 into 6866b61d94de48f7a8e835cf2bc5a9ce(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:37,062 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:37,062 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/B, priority=12, startTime=1732295676988; duration=0sec 2024-11-22T17:14:37,062 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:37,062 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/A, priority=12, startTime=1732295676988; duration=0sec 2024-11-22T17:14:37,062 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:37,062 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:B 2024-11-22T17:14:37,063 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:37,063 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:A 2024-11-22T17:14:37,063 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:37,065 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:37,065 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/C is initiating minor compaction (all files) 2024-11-22T17:14:37,066 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/C in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:37,066 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/43828007be9d4df495c7879a55ebac49, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/6908076924e7454aaa3c1826285e745b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/2b40ecf216234f32866904ee3309923f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/cf85914265dc441cae2b97794d438acd] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=48.8 K 2024-11-22T17:14:37,067 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 43828007be9d4df495c7879a55ebac49, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732295672974 2024-11-22T17:14:37,067 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 6908076924e7454aaa3c1826285e745b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1732295673124 2024-11-22T17:14:37,068 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b40ecf216234f32866904ee3309923f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732295674311 2024-11-22T17:14:37,068 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting cf85914265dc441cae2b97794d438acd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1732295675384 2024-11-22T17:14:37,081 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#C#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:37,082 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/5b55381adb174eaaa106dd7406a92014 is 50, key is test_row_0/C:col10/1732295675384/Put/seqid=0 2024-11-22T17:14:37,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741914_1090 (size=13153) 2024-11-22T17:14:37,098 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/5b55381adb174eaaa106dd7406a92014 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5b55381adb174eaaa106dd7406a92014 2024-11-22T17:14:37,109 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/C of c70a0885ec0e4f41398ae6890c8dabe5 into 5b55381adb174eaaa106dd7406a92014(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:37,109 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:37,109 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/C, priority=12, startTime=1732295676989; duration=0sec 2024-11-22T17:14:37,110 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:37,110 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:C 2024-11-22T17:14:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:37,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:14:37,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:37,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:37,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:37,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:37,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:37,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:37,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/146f05fcbfb64a6aa80241c62a0045e6 is 50, key is test_row_0/A:col10/1732295677174/Put/seqid=0 2024-11-22T17:14:37,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741915_1091 (size=12301) 2024-11-22T17:14:37,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/146f05fcbfb64a6aa80241c62a0045e6 2024-11-22T17:14:37,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/1860d9c064284b47ba67b2c88c216b32 is 50, key is test_row_0/B:col10/1732295677174/Put/seqid=0 2024-11-22T17:14:37,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741916_1092 (size=12301) 2024-11-22T17:14:37,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295677221, max timestamp=1732295677233Num uniques 859; 2024-11-22T17:14:37,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295677221, max timestamp=1732295677233Num uniques 859; 2024-11-22T17:14:37,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:37,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:37,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295677221, max timestamp=1732295677233Num uniques 859; 2024-11-22T17:14:37,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:37,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295737295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:37,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295737401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:37,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:37,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:37,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T17:14:37,522 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:37,522 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:14:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-22T17:14:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T17:14:37,524 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:14:37,525 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:14:37,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:14:37,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295737551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:37,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295737551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:37,553 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in c70a0885ec0e4f41398ae6890c8dabe5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:14:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:37,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:37,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:37,553 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_0' in c70a0885ec0e4f41398ae6890c8dabe5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:14:37,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:37,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:37,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:37,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295737612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:37,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/1860d9c064284b47ba67b2c88c216b32 2024-11-22T17:14:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T17:14:37,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/7644503e6ec440ce8d0dec5563a529d1 is 50, key is test_row_0/C:col10/1732295677174/Put/seqid=0 2024-11-22T17:14:37,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741917_1093 (size=12301) 2024-11-22T17:14:37,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T17:14:37,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:37,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:37,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:37,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:37,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:37,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T17:14:37,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:37,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:37,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:37,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:37,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T17:14:37,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295737922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:37,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:37,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:37,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:37,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T17:14:37,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:37,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:38,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/7644503e6ec440ce8d0dec5563a529d1 2024-11-22T17:14:38,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/146f05fcbfb64a6aa80241c62a0045e6 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/146f05fcbfb64a6aa80241c62a0045e6 2024-11-22T17:14:38,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/146f05fcbfb64a6aa80241c62a0045e6, entries=150, sequenceid=622, filesize=12.0 K 2024-11-22T17:14:38,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/1860d9c064284b47ba67b2c88c216b32 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/1860d9c064284b47ba67b2c88c216b32 2024-11-22T17:14:38,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/1860d9c064284b47ba67b2c88c216b32, entries=150, sequenceid=622, filesize=12.0 K 2024-11-22T17:14:38,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/7644503e6ec440ce8d0dec5563a529d1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/7644503e6ec440ce8d0dec5563a529d1 2024-11-22T17:14:38,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/7644503e6ec440ce8d0dec5563a529d1, entries=150, sequenceid=622, filesize=12.0 K 2024-11-22T17:14:38,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c70a0885ec0e4f41398ae6890c8dabe5 in 889ms, sequenceid=622, compaction requested=false 2024-11-22T17:14:38,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:38,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T17:14:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:38,138 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-22T17:14:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:38,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/004845bf272b43ae892fa5aa1d6e2d73 is 50, key is test_row_0/A:col10/1732295677292/Put/seqid=0 2024-11-22T17:14:38,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741918_1094 (size=12301) 2024-11-22T17:14:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T17:14:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:38,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:38,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295738479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:38,548 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/004845bf272b43ae892fa5aa1d6e2d73 2024-11-22T17:14:38,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/dd28ee0a753b475498eba72435059bd3 is 50, key is test_row_0/B:col10/1732295677292/Put/seqid=0 2024-11-22T17:14:38,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741919_1095 (size=12301) 2024-11-22T17:14:38,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295738591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T17:14:38,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295738801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:38,970 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/dd28ee0a753b475498eba72435059bd3 2024-11-22T17:14:38,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/0109c73f78db4e2992326fbb6991f8c4 is 50, key is test_row_0/C:col10/1732295677292/Put/seqid=0 2024-11-22T17:14:38,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741920_1096 (size=12301) 2024-11-22T17:14:39,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295739112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:39,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:39,387 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/0109c73f78db4e2992326fbb6991f8c4 2024-11-22T17:14:39,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34042 deadline: 1732295739431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:39,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34056 deadline: 1732295739431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:39,434 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in c70a0885ec0e4f41398ae6890c8dabe5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:14:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:39,434 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in c70a0885ec0e4f41398ae6890c8dabe5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:14:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:39,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/004845bf272b43ae892fa5aa1d6e2d73 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/004845bf272b43ae892fa5aa1d6e2d73 2024-11-22T17:14:39,461 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/004845bf272b43ae892fa5aa1d6e2d73, entries=150, sequenceid=658, filesize=12.0 K 2024-11-22T17:14:39,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/dd28ee0a753b475498eba72435059bd3 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/dd28ee0a753b475498eba72435059bd3 2024-11-22T17:14:39,469 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/dd28ee0a753b475498eba72435059bd3, entries=150, sequenceid=658, filesize=12.0 K 2024-11-22T17:14:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/0109c73f78db4e2992326fbb6991f8c4 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/0109c73f78db4e2992326fbb6991f8c4 2024-11-22T17:14:39,477 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/0109c73f78db4e2992326fbb6991f8c4, entries=150, sequenceid=658, filesize=12.0 K 2024-11-22T17:14:39,478 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for c70a0885ec0e4f41398ae6890c8dabe5 in 1340ms, sequenceid=658, compaction requested=true 2024-11-22T17:14:39,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:39,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:39,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-22T17:14:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-22T17:14:39,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-22T17:14:39,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9550 sec 2024-11-22T17:14:39,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.9610 sec 2024-11-22T17:14:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:39,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T17:14:39,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:39,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:39,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:39,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:39,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:39,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:39,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/b7d71ce89e74497fbe1dffdf9cbc9f69 is 50, key is test_row_0/A:col10/1732295678466/Put/seqid=0 2024-11-22T17:14:39,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741921_1097 (size=14741) 2024-11-22T17:14:39,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295679623, max timestamp=1732295679645Num uniques 859; 2024-11-22T17:14:39,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295679623, max timestamp=1732295679645Num uniques 859; 2024-11-22T17:14:39,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295679623, max timestamp=1732295679645Num uniques 859; 2024-11-22T17:14:39,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:39,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:39,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:14:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T17:14:39,682 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:39,682 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:14:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-22T17:14:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T17:14:39,685 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:14:39,686 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:14:39,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:14:39,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295739682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:39,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295739683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:39,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295739685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T17:14:39,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:39,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:39,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:39,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:39,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:39,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:39,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295739875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:39,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:39,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295739875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:39,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295739876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:39,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/b7d71ce89e74497fbe1dffdf9cbc9f69 2024-11-22T17:14:39,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/403f5dd6d0404d088ba907e9e5c391cf is 50, key is test_row_0/B:col10/1732295678466/Put/seqid=0 2024-11-22T17:14:39,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741922_1098 (size=12301) 2024-11-22T17:14:40,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:40,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:40,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T17:14:40,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295740091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:40,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295740091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295740091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:40,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:40,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:40,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:40,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:40,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T17:14:40,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/403f5dd6d0404d088ba907e9e5c391cf 2024-11-22T17:14:40,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/5d1e62a1ba914cffad1a276ccbf93038 is 50, key is test_row_0/C:col10/1732295678466/Put/seqid=0 2024-11-22T17:14:40,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295740401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:40,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295740402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:40,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:40,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295740402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:40,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:40,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:40,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:40,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741923_1099 (size=12301) 2024-11-22T17:14:40,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:40,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:40,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:40,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:40,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:40,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:40,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/5d1e62a1ba914cffad1a276ccbf93038 2024-11-22T17:14:40,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/b7d71ce89e74497fbe1dffdf9cbc9f69 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/b7d71ce89e74497fbe1dffdf9cbc9f69 2024-11-22T17:14:40,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/b7d71ce89e74497fbe1dffdf9cbc9f69, entries=200, sequenceid=669, filesize=14.4 K 2024-11-22T17:14:40,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/403f5dd6d0404d088ba907e9e5c391cf as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/403f5dd6d0404d088ba907e9e5c391cf 2024-11-22T17:14:40,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/403f5dd6d0404d088ba907e9e5c391cf, entries=150, sequenceid=669, filesize=12.0 K 2024-11-22T17:14:40,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/5d1e62a1ba914cffad1a276ccbf93038 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5d1e62a1ba914cffad1a276ccbf93038 2024-11-22T17:14:40,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5d1e62a1ba914cffad1a276ccbf93038, entries=150, sequenceid=669, filesize=12.0 K 2024-11-22T17:14:40,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for c70a0885ec0e4f41398ae6890c8dabe5 in 1282ms, sequenceid=669, compaction requested=true 2024-11-22T17:14:40,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:40,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:14:40,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:40,848 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:40,848 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:40,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:14:40,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:40,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c70a0885ec0e4f41398ae6890c8dabe5:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:14:40,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:40,858 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:40,858 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/A is initiating minor compaction (all files) 2024-11-22T17:14:40,858 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/A in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,858 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/8f33d475fa094832b6bf6cf30cc5f47b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/146f05fcbfb64a6aa80241c62a0045e6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/004845bf272b43ae892fa5aa1d6e2d73, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/b7d71ce89e74497fbe1dffdf9cbc9f69] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=51.3 K 2024-11-22T17:14:40,859 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:40,859 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/B is initiating minor compaction (all files) 2024-11-22T17:14:40,859 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/B in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,859 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6866b61d94de48f7a8e835cf2bc5a9ce, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/1860d9c064284b47ba67b2c88c216b32, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/dd28ee0a753b475498eba72435059bd3, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/403f5dd6d0404d088ba907e9e5c391cf] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=48.9 K 2024-11-22T17:14:40,860 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f33d475fa094832b6bf6cf30cc5f47b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1732295675384 2024-11-22T17:14:40,860 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 6866b61d94de48f7a8e835cf2bc5a9ce, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1732295675384 2024-11-22T17:14:40,860 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 146f05fcbfb64a6aa80241c62a0045e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1732295676522 2024-11-22T17:14:40,860 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 1860d9c064284b47ba67b2c88c216b32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1732295676522 2024-11-22T17:14:40,861 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 004845bf272b43ae892fa5aa1d6e2d73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1732295677280 2024-11-22T17:14:40,861 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting dd28ee0a753b475498eba72435059bd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1732295677280 2024-11-22T17:14:40,861 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7d71ce89e74497fbe1dffdf9cbc9f69, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1732295678466 2024-11-22T17:14:40,862 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 403f5dd6d0404d088ba907e9e5c391cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1732295678466 2024-11-22T17:14:40,875 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:40,878 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/6177020eaabd4f409815e6ec155d6ee8 is 50, key is test_row_0/B:col10/1732295678466/Put/seqid=0 2024-11-22T17:14:40,879 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#A#compaction#88 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:40,879 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/426c234c9111406482ae0a27a48a8956 is 50, key is test_row_0/A:col10/1732295678466/Put/seqid=0 2024-11-22T17:14:40,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741924_1100 (size=13289) 2024-11-22T17:14:40,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741925_1101 (size=13289) 2024-11-22T17:14:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T17:14:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:40,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-22T17:14:40,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:40,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:40,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:40,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:40,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:40,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:40,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/a00576187b6d4895b2c26a12ea6932a4 is 50, key is test_row_0/A:col10/1732295679673/Put/seqid=0 2024-11-22T17:14:40,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741926_1102 (size=14741) 2024-11-22T17:14:40,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295740933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:40,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295740933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:40,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:40,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295740943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:40,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:40,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:40,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:40,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295741051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:41,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295741051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:41,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295741051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:41,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:41,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:41,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:41,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:41,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:41,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:41,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:41,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:41,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295741261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295741261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295741261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:41,292 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/6177020eaabd4f409815e6ec155d6ee8 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6177020eaabd4f409815e6ec155d6ee8 2024-11-22T17:14:41,294 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/426c234c9111406482ae0a27a48a8956 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/426c234c9111406482ae0a27a48a8956 2024-11-22T17:14:41,300 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/B of c70a0885ec0e4f41398ae6890c8dabe5 into 6177020eaabd4f409815e6ec155d6ee8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:41,300 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:41,300 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/B, priority=12, startTime=1732295680848; duration=0sec 2024-11-22T17:14:41,300 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:41,300 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:B 2024-11-22T17:14:41,300 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:41,301 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/A of c70a0885ec0e4f41398ae6890c8dabe5 into 426c234c9111406482ae0a27a48a8956(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:41,301 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:41,301 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/A, priority=12, startTime=1732295680848; duration=0sec 2024-11-22T17:14:41,301 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:41,301 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:A 2024-11-22T17:14:41,302 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:41,303 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): c70a0885ec0e4f41398ae6890c8dabe5/C is initiating minor compaction (all files) 2024-11-22T17:14:41,303 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c70a0885ec0e4f41398ae6890c8dabe5/C in TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:41,303 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5b55381adb174eaaa106dd7406a92014, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/7644503e6ec440ce8d0dec5563a529d1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/0109c73f78db4e2992326fbb6991f8c4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5d1e62a1ba914cffad1a276ccbf93038] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp, totalSize=48.9 K 2024-11-22T17:14:41,303 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b55381adb174eaaa106dd7406a92014, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1732295675384 2024-11-22T17:14:41,304 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 7644503e6ec440ce8d0dec5563a529d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1732295676522 2024-11-22T17:14:41,304 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 0109c73f78db4e2992326fbb6991f8c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1732295677280 2024-11-22T17:14:41,304 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1e62a1ba914cffad1a276ccbf93038, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1732295678466 2024-11-22T17:14:41,314 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c70a0885ec0e4f41398ae6890c8dabe5#C#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:41,315 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/c108a849f53f4427ae127516db98507d is 50, key is test_row_0/C:col10/1732295678466/Put/seqid=0 2024-11-22T17:14:41,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741927_1103 (size=13289) 2024-11-22T17:14:41,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=706 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/a00576187b6d4895b2c26a12ea6932a4 2024-11-22T17:14:41,325 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/c108a849f53f4427ae127516db98507d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/c108a849f53f4427ae127516db98507d 2024-11-22T17:14:41,331 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c70a0885ec0e4f41398ae6890c8dabe5/C of c70a0885ec0e4f41398ae6890c8dabe5 into c108a849f53f4427ae127516db98507d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:41,331 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:41,331 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., storeName=c70a0885ec0e4f41398ae6890c8dabe5/C, priority=12, startTime=1732295680848; duration=0sec 2024-11-22T17:14:41,331 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:41,331 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c70a0885ec0e4f41398ae6890c8dabe5:C 2024-11-22T17:14:41,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/184ccbd29b914dcbb3c8953433fcacee is 50, key is test_row_0/B:col10/1732295679673/Put/seqid=0 2024-11-22T17:14:41,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741928_1104 (size=12301) 2024-11-22T17:14:41,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:41,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:41,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:41,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:41,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:41,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:41,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:41,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,567 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-22T17:14:41,567 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:14:41,569 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-22T17:14:41,569 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:14:41,571 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-22T17:14:41,572 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:14:41,572 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-22T17:14:41,572 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:14:41,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:41,572 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:41,572 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:41,572 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:41,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295741572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295741572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:41,572 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:41,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:41,572 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295741572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:41,573 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:41,573 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:41,573 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:14:41,573 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:14:41,573 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:41,573 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:14:41,573 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:14:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:41,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:41,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:41,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:41,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:41,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=706 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/184ccbd29b914dcbb3c8953433fcacee 2024-11-22T17:14:41,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/e5658f6039554b4a87d931dce09ccb0d is 50, key is test_row_0/C:col10/1732295679673/Put/seqid=0 2024-11-22T17:14:41,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741929_1105 (size=12301) 2024-11-22T17:14:41,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:41,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:41,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:41,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:41,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T17:14:42,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:42,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:42,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:42,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:42,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:42,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:42,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:42,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:42,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34022 deadline: 1732295742082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33990 deadline: 1732295742082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34052 deadline: 1732295742082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:42,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:42,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:42,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:42,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:14:42,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=706 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/e5658f6039554b4a87d931dce09ccb0d 2024-11-22T17:14:42,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/a00576187b6d4895b2c26a12ea6932a4 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/a00576187b6d4895b2c26a12ea6932a4 2024-11-22T17:14:42,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/a00576187b6d4895b2c26a12ea6932a4, entries=200, sequenceid=706, filesize=14.4 K 2024-11-22T17:14:42,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/184ccbd29b914dcbb3c8953433fcacee as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/184ccbd29b914dcbb3c8953433fcacee 2024-11-22T17:14:42,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:42,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:42,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:42,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:42,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:42,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:42,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/184ccbd29b914dcbb3c8953433fcacee, entries=150, sequenceid=706, filesize=12.0 K 2024-11-22T17:14:42,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/e5658f6039554b4a87d931dce09ccb0d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/e5658f6039554b4a87d931dce09ccb0d 2024-11-22T17:14:42,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/e5658f6039554b4a87d931dce09ccb0d, entries=150, sequenceid=706, filesize=12.0 K 2024-11-22T17:14:42,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for c70a0885ec0e4f41398ae6890c8dabe5 in 1277ms, sequenceid=706, compaction requested=false 2024-11-22T17:14:42,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:42,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T17:14:42,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:42,337 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:14:42,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:42,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:42,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:42,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:42,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:42,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/9aadacaf5936470cbf4ac9e5f0b13c2a is 50, key is test_row_0/A:col10/1732295680932/Put/seqid=0 2024-11-22T17:14:42,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741930_1106 (size=12301) 2024-11-22T17:14:42,751 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=719 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/9aadacaf5936470cbf4ac9e5f0b13c2a 2024-11-22T17:14:42,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/325154a39dd04cb2a8eb52410b1758f3 is 50, key is test_row_0/B:col10/1732295680932/Put/seqid=0 2024-11-22T17:14:42,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741931_1107 (size=12301) 2024-11-22T17:14:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:43,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. as already flushing 2024-11-22T17:14:43,096 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-22T17:14:43,097 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:14:43,097 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:43,097 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:43,097 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:14:43,104 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-22T17:14:43,104 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-22T17:14:43,104 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:14:43,104 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:14:43,104 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:43,104 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:43,105 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:43,105 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:43,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:14:43,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:14:43,170 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=719 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/325154a39dd04cb2a8eb52410b1758f3 2024-11-22T17:14:43,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/7bdf67382a3b4445ad2706dec0d95852 is 50, key is test_row_0/C:col10/1732295680932/Put/seqid=0 2024-11-22T17:14:43,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741932_1108 (size=12301) 2024-11-22T17:14:43,515 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T17:14:43,586 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=719 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/7bdf67382a3b4445ad2706dec0d95852 2024-11-22T17:14:43,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/9aadacaf5936470cbf4ac9e5f0b13c2a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/9aadacaf5936470cbf4ac9e5f0b13c2a 2024-11-22T17:14:43,601 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/9aadacaf5936470cbf4ac9e5f0b13c2a, entries=150, sequenceid=719, filesize=12.0 K 2024-11-22T17:14:43,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/325154a39dd04cb2a8eb52410b1758f3 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/325154a39dd04cb2a8eb52410b1758f3 2024-11-22T17:14:43,607 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/325154a39dd04cb2a8eb52410b1758f3, entries=150, sequenceid=719, filesize=12.0 K 2024-11-22T17:14:43,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/7bdf67382a3b4445ad2706dec0d95852 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/7bdf67382a3b4445ad2706dec0d95852 2024-11-22T17:14:43,612 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/7bdf67382a3b4445ad2706dec0d95852, entries=150, sequenceid=719, filesize=12.0 K 2024-11-22T17:14:43,613 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=20.13 KB/20610 for c70a0885ec0e4f41398ae6890c8dabe5 in 1275ms, sequenceid=719, compaction requested=true 2024-11-22T17:14:43,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for c70a0885ec0e4f41398ae6890c8dabe5: 2024-11-22T17:14:43,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:43,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-22T17:14:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-22T17:14:43,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-22T17:14:43,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9280 sec 2024-11-22T17:14:43,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 3.9340 sec 2024-11-22T17:14:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T17:14:43,932 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:49,457 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-22T17:14:49,458 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:14:49,458 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:49,458 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:49,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:14:49,495 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-22T17:14:49,495 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:14:49,495 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:49,495 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:49,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:14:49,496 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T17:14:49,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-22T17:14:49,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-11-22T17:14:49,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-22T17:14:49,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-22T17:14:49,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 194 2024-11-22T17:14:49,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T17:14:49,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2523 2024-11-22T17:14:49,498 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2558 2024-11-22T17:14:49,498 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T17:14:49,498 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1122 2024-11-22T17:14:49,498 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3362 rows 2024-11-22T17:14:49,498 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1115 2024-11-22T17:14:49,498 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3341 rows 2024-11-22T17:14:49,499 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T17:14:49,499 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T17:14:49,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:49,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:49,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:14:49,506 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T17:14:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T17:14:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T17:14:49,513 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295689512"}]},"ts":"1732295689512"} 2024-11-22T17:14:49,514 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T17:14:49,515 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T17:14:49,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T17:14:49,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c70a0885ec0e4f41398ae6890c8dabe5, UNASSIGN}] 2024-11-22T17:14:49,521 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c70a0885ec0e4f41398ae6890c8dabe5, UNASSIGN 2024-11-22T17:14:49,522 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=c70a0885ec0e4f41398ae6890c8dabe5, regionState=CLOSING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:49,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c70a0885ec0e4f41398ae6890c8dabe5, UNASSIGN because future has completed 2024-11-22T17:14:49,525 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T17:14:49,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:14:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T17:14:49,687 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:49,688 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T17:14:49,689 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing c70a0885ec0e4f41398ae6890c8dabe5, disabling compactions & flushes 2024-11-22T17:14:49,689 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:49,690 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:49,690 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. after waiting 0 ms 2024-11-22T17:14:49,690 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:49,690 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing c70a0885ec0e4f41398ae6890c8dabe5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-22T17:14:49,691 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=A 2024-11-22T17:14:49,691 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:49,691 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=B 2024-11-22T17:14:49,691 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:49,691 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c70a0885ec0e4f41398ae6890c8dabe5, store=C 2024-11-22T17:14:49,691 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:49,696 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/844d3b50778b4a93ba3b7bfd0be23d06 is 50, key is test_row_0/A:col10/1732295683093/Put/seqid=0 2024-11-22T17:14:49,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741933_1109 (size=12301) 2024-11-22T17:14:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T17:14:50,103 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/844d3b50778b4a93ba3b7bfd0be23d06 2024-11-22T17:14:50,119 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/f21d394890f048edab16e04035b43863 is 50, key is test_row_0/B:col10/1732295683093/Put/seqid=0 2024-11-22T17:14:50,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741934_1110 (size=12301) 2024-11-22T17:14:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T17:14:50,524 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/f21d394890f048edab16e04035b43863 2024-11-22T17:14:50,536 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/b48c8505676e4e059ed3b73741f58bb2 is 50, key is test_row_0/C:col10/1732295683093/Put/seqid=0 2024-11-22T17:14:50,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741935_1111 (size=12301) 2024-11-22T17:14:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T17:14:50,942 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/b48c8505676e4e059ed3b73741f58bb2 2024-11-22T17:14:50,954 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/A/844d3b50778b4a93ba3b7bfd0be23d06 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/844d3b50778b4a93ba3b7bfd0be23d06 2024-11-22T17:14:50,961 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/844d3b50778b4a93ba3b7bfd0be23d06, entries=150, sequenceid=727, filesize=12.0 K 2024-11-22T17:14:50,962 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/B/f21d394890f048edab16e04035b43863 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/f21d394890f048edab16e04035b43863 2024-11-22T17:14:50,967 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/f21d394890f048edab16e04035b43863, entries=150, sequenceid=727, filesize=12.0 K 2024-11-22T17:14:50,968 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/.tmp/C/b48c8505676e4e059ed3b73741f58bb2 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/b48c8505676e4e059ed3b73741f58bb2 2024-11-22T17:14:50,974 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/b48c8505676e4e059ed3b73741f58bb2, entries=150, sequenceid=727, filesize=12.0 K 2024-11-22T17:14:50,975 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c70a0885ec0e4f41398ae6890c8dabe5 in 1284ms, sequenceid=727, compaction requested=true 2024-11-22T17:14:50,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/efb4a87b0a6644dc98085473cc58d2b4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/bf6918868f4f420ba8f19078abd6ef01, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/6f486ddcad2b44cebacf6de8fa067cad, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/c6e5b27aa6014a998e3b31aa4d56e312, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/9594292a1792425bbe77baf95b9756f1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/59fd11c2b9034f679e80a8f6960d57ad, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/98eb3d7fea0f4015b4960cbbe918f335, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/d24f3821a7864876b4c1051131cace27, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/bfab417825734ce4bfd07758330d085b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/57082167416a4ee0ae8164dde7e0f40f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/1f4791e1e4a447099016cd3f6c335822, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/e634a0b22a57493abdd494a5eacf32ea, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/af9a2b27b1e94ce29693161167af6aa9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/dda020ff1e274d38a5905aa630f48c49, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/b9be9afff0094859adec22f044fec6aa, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/fb3a063ab5bf42909a6dc38b53245adb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/24791c8083ba4c8db00d7a7a57f66c3c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/8556d592606048c8b7feab676f217470, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/f05ebd86d8aa4ebf9bf8ab9c8c36819f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/eb4f767eda514da28534012f0adf81b2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/95cc26b4455f4c77acf998f0ceaa3e49, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/81fed2f9cc504bdc82499d7e50c41014, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/eed8274a39804df1bbc6a6cc5db32a46, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/a913c931cfe54b3e87b13fab800b14c0, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/ab22b9d68a83471c8d42556e82de74e8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/8f33d475fa094832b6bf6cf30cc5f47b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/146f05fcbfb64a6aa80241c62a0045e6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/004845bf272b43ae892fa5aa1d6e2d73, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/b7d71ce89e74497fbe1dffdf9cbc9f69] to archive 2024-11-22T17:14:50,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:14:50,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/efb4a87b0a6644dc98085473cc58d2b4 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/efb4a87b0a6644dc98085473cc58d2b4 2024-11-22T17:14:50,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/bf6918868f4f420ba8f19078abd6ef01 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/bf6918868f4f420ba8f19078abd6ef01 2024-11-22T17:14:50,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/6f486ddcad2b44cebacf6de8fa067cad to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/6f486ddcad2b44cebacf6de8fa067cad 2024-11-22T17:14:50,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/c6e5b27aa6014a998e3b31aa4d56e312 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/c6e5b27aa6014a998e3b31aa4d56e312 2024-11-22T17:14:50,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/9594292a1792425bbe77baf95b9756f1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/9594292a1792425bbe77baf95b9756f1 2024-11-22T17:14:50,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/59fd11c2b9034f679e80a8f6960d57ad to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/59fd11c2b9034f679e80a8f6960d57ad 2024-11-22T17:14:50,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/98eb3d7fea0f4015b4960cbbe918f335 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/98eb3d7fea0f4015b4960cbbe918f335 2024-11-22T17:14:50,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/d24f3821a7864876b4c1051131cace27 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/d24f3821a7864876b4c1051131cace27 2024-11-22T17:14:50,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/bfab417825734ce4bfd07758330d085b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/bfab417825734ce4bfd07758330d085b 2024-11-22T17:14:50,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/57082167416a4ee0ae8164dde7e0f40f to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/57082167416a4ee0ae8164dde7e0f40f 2024-11-22T17:14:51,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/1f4791e1e4a447099016cd3f6c335822 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/1f4791e1e4a447099016cd3f6c335822 2024-11-22T17:14:51,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/e634a0b22a57493abdd494a5eacf32ea to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/e634a0b22a57493abdd494a5eacf32ea 2024-11-22T17:14:51,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/af9a2b27b1e94ce29693161167af6aa9 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/af9a2b27b1e94ce29693161167af6aa9 2024-11-22T17:14:51,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/dda020ff1e274d38a5905aa630f48c49 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/dda020ff1e274d38a5905aa630f48c49 2024-11-22T17:14:51,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/b9be9afff0094859adec22f044fec6aa to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/b9be9afff0094859adec22f044fec6aa 2024-11-22T17:14:51,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/fb3a063ab5bf42909a6dc38b53245adb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/fb3a063ab5bf42909a6dc38b53245adb 2024-11-22T17:14:51,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/24791c8083ba4c8db00d7a7a57f66c3c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/24791c8083ba4c8db00d7a7a57f66c3c 2024-11-22T17:14:51,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/8556d592606048c8b7feab676f217470 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/8556d592606048c8b7feab676f217470 2024-11-22T17:14:51,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/f05ebd86d8aa4ebf9bf8ab9c8c36819f to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/f05ebd86d8aa4ebf9bf8ab9c8c36819f 2024-11-22T17:14:51,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/eb4f767eda514da28534012f0adf81b2 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/eb4f767eda514da28534012f0adf81b2 2024-11-22T17:14:51,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/95cc26b4455f4c77acf998f0ceaa3e49 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/95cc26b4455f4c77acf998f0ceaa3e49 2024-11-22T17:14:51,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/81fed2f9cc504bdc82499d7e50c41014 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/81fed2f9cc504bdc82499d7e50c41014 2024-11-22T17:14:51,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/eed8274a39804df1bbc6a6cc5db32a46 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/eed8274a39804df1bbc6a6cc5db32a46 2024-11-22T17:14:51,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/a913c931cfe54b3e87b13fab800b14c0 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/a913c931cfe54b3e87b13fab800b14c0 2024-11-22T17:14:51,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/ab22b9d68a83471c8d42556e82de74e8 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/ab22b9d68a83471c8d42556e82de74e8 2024-11-22T17:14:51,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/8f33d475fa094832b6bf6cf30cc5f47b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/8f33d475fa094832b6bf6cf30cc5f47b 2024-11-22T17:14:51,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/146f05fcbfb64a6aa80241c62a0045e6 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/146f05fcbfb64a6aa80241c62a0045e6 2024-11-22T17:14:51,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/004845bf272b43ae892fa5aa1d6e2d73 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/004845bf272b43ae892fa5aa1d6e2d73 2024-11-22T17:14:51,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/b7d71ce89e74497fbe1dffdf9cbc9f69 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/b7d71ce89e74497fbe1dffdf9cbc9f69 2024-11-22T17:14:51,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/913237a97906446bab93b1a2cc4d6f7e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/baec977fffa446b69e2d0d879b70f3a9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/82680d69cf6e43afa382cfa67fb7fc98, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/a1247f6fcacf49d0a0242ef53a40710b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/7eaf27ef7f1548e583e2a3a2dc6dbc41, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/4a1cf233faad4279a5905af44a32a471, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/3cb686fab3764eefb04e90dd146b680b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/100c6aaffcdd4a39ae3008f2bf44618d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/85554f4438ee44d2bc871f18772d6150, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/99487b642ea5469790a079479ef79df1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/5a2fdf432a34444584036ae512e1617c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/7517bbf440a243f9aa54398167df8cb5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/61c9148a7d754420986893b8989ebd37, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/39edb64c590340d48f8ba7b7dbf462a9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6850a7fc032c4f71bc93fc21ff4385dc, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/32a9b05031bc4539b1a1a7106f3acd5e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/5a50818973344beeacf67e28fb108d2d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/07bebd19f1974b329e5586f46bfe1bbd, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/0486b51b3db3488fa567830567c3370b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/a1883077a6544c60a0b50c113ad3cecb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/995dd5b00f1349d7aa721158d9588fde, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/b11c5e8994a3433d8dce0bef46a8c869, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/4d8d52c9402344169af6751fddb1ad2a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/af6db8aea548404c83343a5d2f533f55, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6866b61d94de48f7a8e835cf2bc5a9ce, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/13200f7d08e2476fb9da91ec9b578d4e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/1860d9c064284b47ba67b2c88c216b32, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/dd28ee0a753b475498eba72435059bd3, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/403f5dd6d0404d088ba907e9e5c391cf] to archive 2024-11-22T17:14:51,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:14:51,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/913237a97906446bab93b1a2cc4d6f7e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/913237a97906446bab93b1a2cc4d6f7e 2024-11-22T17:14:51,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/baec977fffa446b69e2d0d879b70f3a9 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/baec977fffa446b69e2d0d879b70f3a9 2024-11-22T17:14:51,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/82680d69cf6e43afa382cfa67fb7fc98 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/82680d69cf6e43afa382cfa67fb7fc98 2024-11-22T17:14:51,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/a1247f6fcacf49d0a0242ef53a40710b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/a1247f6fcacf49d0a0242ef53a40710b 2024-11-22T17:14:51,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/7eaf27ef7f1548e583e2a3a2dc6dbc41 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/7eaf27ef7f1548e583e2a3a2dc6dbc41 2024-11-22T17:14:51,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/4a1cf233faad4279a5905af44a32a471 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/4a1cf233faad4279a5905af44a32a471 2024-11-22T17:14:51,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/3cb686fab3764eefb04e90dd146b680b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/3cb686fab3764eefb04e90dd146b680b 2024-11-22T17:14:51,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/100c6aaffcdd4a39ae3008f2bf44618d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/100c6aaffcdd4a39ae3008f2bf44618d 2024-11-22T17:14:51,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/85554f4438ee44d2bc871f18772d6150 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/85554f4438ee44d2bc871f18772d6150 2024-11-22T17:14:51,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/99487b642ea5469790a079479ef79df1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/99487b642ea5469790a079479ef79df1 2024-11-22T17:14:51,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/5a2fdf432a34444584036ae512e1617c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/5a2fdf432a34444584036ae512e1617c 2024-11-22T17:14:51,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/7517bbf440a243f9aa54398167df8cb5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/7517bbf440a243f9aa54398167df8cb5 2024-11-22T17:14:51,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/61c9148a7d754420986893b8989ebd37 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/61c9148a7d754420986893b8989ebd37 2024-11-22T17:14:51,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/39edb64c590340d48f8ba7b7dbf462a9 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/39edb64c590340d48f8ba7b7dbf462a9 2024-11-22T17:14:51,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6850a7fc032c4f71bc93fc21ff4385dc to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6850a7fc032c4f71bc93fc21ff4385dc 2024-11-22T17:14:51,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/32a9b05031bc4539b1a1a7106f3acd5e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/32a9b05031bc4539b1a1a7106f3acd5e 2024-11-22T17:14:51,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/5a50818973344beeacf67e28fb108d2d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/5a50818973344beeacf67e28fb108d2d 2024-11-22T17:14:51,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/07bebd19f1974b329e5586f46bfe1bbd to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/07bebd19f1974b329e5586f46bfe1bbd 2024-11-22T17:14:51,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/0486b51b3db3488fa567830567c3370b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/0486b51b3db3488fa567830567c3370b 2024-11-22T17:14:51,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/a1883077a6544c60a0b50c113ad3cecb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/a1883077a6544c60a0b50c113ad3cecb 2024-11-22T17:14:51,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/995dd5b00f1349d7aa721158d9588fde to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/995dd5b00f1349d7aa721158d9588fde 2024-11-22T17:14:51,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/b11c5e8994a3433d8dce0bef46a8c869 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/b11c5e8994a3433d8dce0bef46a8c869 2024-11-22T17:14:51,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/4d8d52c9402344169af6751fddb1ad2a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/4d8d52c9402344169af6751fddb1ad2a 2024-11-22T17:14:51,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/af6db8aea548404c83343a5d2f533f55 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/af6db8aea548404c83343a5d2f533f55 2024-11-22T17:14:51,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6866b61d94de48f7a8e835cf2bc5a9ce to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6866b61d94de48f7a8e835cf2bc5a9ce 2024-11-22T17:14:51,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/13200f7d08e2476fb9da91ec9b578d4e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/13200f7d08e2476fb9da91ec9b578d4e 2024-11-22T17:14:51,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/1860d9c064284b47ba67b2c88c216b32 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/1860d9c064284b47ba67b2c88c216b32 2024-11-22T17:14:51,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/dd28ee0a753b475498eba72435059bd3 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/dd28ee0a753b475498eba72435059bd3 2024-11-22T17:14:51,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/403f5dd6d0404d088ba907e9e5c391cf to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/403f5dd6d0404d088ba907e9e5c391cf 2024-11-22T17:14:51,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/950737c9a65d4188b03ea22b929cb2de, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/204c0af83a884769968b8783459f183b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/c38071bb8116403c8e2c03b2c207085d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/65964acb58ae488dae2b38d623dad561, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/91bf32e73c7f4e71b255e96b95d06956, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/d77ae0c438664a5da80efd64b433772c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/763a24d0bdf04c8db4b5b801b6e1e6ad, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/fcc23f9bf66c46fd97e4f14a1186bcb1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/3758e469874a417eb1d795038afe3ef8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/ec5d124683a8474993710892f7cc686e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/11842d79f0b74a56aefa5272a335baef, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/e5e1e73e62fc4157bc723eb373b8675d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/4282b33266ce4bcb8ebe5abd8b62b53b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/861e745b7ebc4fde8240a09f4a9a6aaf, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5f179a980a9947c78a847d8e960d502e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/ff8a1b64b7204b2ca60a7bb0debc7ebc, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/2b14eb52abd648a0bbccf44149af5c3c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/d8c17da044e4463697ad4169c274051c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/605d9660f4a94d11ae2ddd2e7b41dcd8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/df9b23c9d22947adaf5d65b4113b8fc3, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/43828007be9d4df495c7879a55ebac49, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/b72c9683f9d24124bf57417a05699567, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/6908076924e7454aaa3c1826285e745b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/2b40ecf216234f32866904ee3309923f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5b55381adb174eaaa106dd7406a92014, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/cf85914265dc441cae2b97794d438acd, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/7644503e6ec440ce8d0dec5563a529d1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/0109c73f78db4e2992326fbb6991f8c4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5d1e62a1ba914cffad1a276ccbf93038] to archive 2024-11-22T17:14:51,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:14:51,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/950737c9a65d4188b03ea22b929cb2de to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/950737c9a65d4188b03ea22b929cb2de 2024-11-22T17:14:51,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/204c0af83a884769968b8783459f183b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/204c0af83a884769968b8783459f183b 2024-11-22T17:14:51,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/c38071bb8116403c8e2c03b2c207085d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/c38071bb8116403c8e2c03b2c207085d 2024-11-22T17:14:51,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/65964acb58ae488dae2b38d623dad561 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/65964acb58ae488dae2b38d623dad561 2024-11-22T17:14:51,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/91bf32e73c7f4e71b255e96b95d06956 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/91bf32e73c7f4e71b255e96b95d06956 2024-11-22T17:14:51,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/d77ae0c438664a5da80efd64b433772c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/d77ae0c438664a5da80efd64b433772c 2024-11-22T17:14:51,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/763a24d0bdf04c8db4b5b801b6e1e6ad to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/763a24d0bdf04c8db4b5b801b6e1e6ad 2024-11-22T17:14:51,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/fcc23f9bf66c46fd97e4f14a1186bcb1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/fcc23f9bf66c46fd97e4f14a1186bcb1 2024-11-22T17:14:51,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/3758e469874a417eb1d795038afe3ef8 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/3758e469874a417eb1d795038afe3ef8 2024-11-22T17:14:51,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/ec5d124683a8474993710892f7cc686e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/ec5d124683a8474993710892f7cc686e 2024-11-22T17:14:51,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/11842d79f0b74a56aefa5272a335baef to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/11842d79f0b74a56aefa5272a335baef 2024-11-22T17:14:51,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/e5e1e73e62fc4157bc723eb373b8675d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/e5e1e73e62fc4157bc723eb373b8675d 2024-11-22T17:14:51,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/4282b33266ce4bcb8ebe5abd8b62b53b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/4282b33266ce4bcb8ebe5abd8b62b53b 2024-11-22T17:14:51,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/861e745b7ebc4fde8240a09f4a9a6aaf to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/861e745b7ebc4fde8240a09f4a9a6aaf 2024-11-22T17:14:51,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5f179a980a9947c78a847d8e960d502e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5f179a980a9947c78a847d8e960d502e 2024-11-22T17:14:51,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/ff8a1b64b7204b2ca60a7bb0debc7ebc to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/ff8a1b64b7204b2ca60a7bb0debc7ebc 2024-11-22T17:14:51,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/2b14eb52abd648a0bbccf44149af5c3c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/2b14eb52abd648a0bbccf44149af5c3c 2024-11-22T17:14:51,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/d8c17da044e4463697ad4169c274051c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/d8c17da044e4463697ad4169c274051c 2024-11-22T17:14:51,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/605d9660f4a94d11ae2ddd2e7b41dcd8 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/605d9660f4a94d11ae2ddd2e7b41dcd8 2024-11-22T17:14:51,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/df9b23c9d22947adaf5d65b4113b8fc3 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/df9b23c9d22947adaf5d65b4113b8fc3 2024-11-22T17:14:51,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/43828007be9d4df495c7879a55ebac49 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/43828007be9d4df495c7879a55ebac49 2024-11-22T17:14:51,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/b72c9683f9d24124bf57417a05699567 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/b72c9683f9d24124bf57417a05699567 2024-11-22T17:14:51,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/6908076924e7454aaa3c1826285e745b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/6908076924e7454aaa3c1826285e745b 2024-11-22T17:14:51,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/2b40ecf216234f32866904ee3309923f to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/2b40ecf216234f32866904ee3309923f 2024-11-22T17:14:51,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5b55381adb174eaaa106dd7406a92014 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5b55381adb174eaaa106dd7406a92014 2024-11-22T17:14:51,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/cf85914265dc441cae2b97794d438acd to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/cf85914265dc441cae2b97794d438acd 2024-11-22T17:14:51,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/7644503e6ec440ce8d0dec5563a529d1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/7644503e6ec440ce8d0dec5563a529d1 2024-11-22T17:14:51,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/0109c73f78db4e2992326fbb6991f8c4 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/0109c73f78db4e2992326fbb6991f8c4 2024-11-22T17:14:51,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5d1e62a1ba914cffad1a276ccbf93038 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/5d1e62a1ba914cffad1a276ccbf93038 2024-11-22T17:14:51,117 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/recovered.edits/730.seqid, newMaxSeqId=730, maxSeqId=1 2024-11-22T17:14:51,118 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. 2024-11-22T17:14:51,119 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for c70a0885ec0e4f41398ae6890c8dabe5: Waiting for close lock at 1732295689689Running coprocessor pre-close hooks at 1732295689689Disabling compacts and flushes for region at 1732295689689Disabling writes for close at 1732295689690 (+1 ms)Obtaining lock to block concurrent updates at 1732295689690Preparing flush snapshotting stores in c70a0885ec0e4f41398ae6890c8dabe5 at 1732295689690Finished memstore snapshotting TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732295689691 (+1 ms)Flushing stores of TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5. at 1732295689692 (+1 ms)Flushing c70a0885ec0e4f41398ae6890c8dabe5/A: creating writer at 1732295689692Flushing c70a0885ec0e4f41398ae6890c8dabe5/A: appending metadata at 1732295689696 (+4 ms)Flushing c70a0885ec0e4f41398ae6890c8dabe5/A: closing flushed file at 1732295689696Flushing c70a0885ec0e4f41398ae6890c8dabe5/B: creating writer at 1732295690114 (+418 ms)Flushing c70a0885ec0e4f41398ae6890c8dabe5/B: appending metadata at 1732295690118 (+4 ms)Flushing c70a0885ec0e4f41398ae6890c8dabe5/B: closing flushed file at 1732295690118Flushing c70a0885ec0e4f41398ae6890c8dabe5/C: creating writer at 1732295690533 (+415 ms)Flushing c70a0885ec0e4f41398ae6890c8dabe5/C: appending metadata at 1732295690535 (+2 ms)Flushing c70a0885ec0e4f41398ae6890c8dabe5/C: closing flushed file at 1732295690535Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48d5be08: reopening flushed file at 1732295690953 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61c1f3b4: reopening flushed file at 1732295690961 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c8b1b83: reopening flushed file at 1732295690967 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c70a0885ec0e4f41398ae6890c8dabe5 in 1284ms, sequenceid=727, compaction requested=true at 1732295690975 (+8 ms)Writing region close event to WAL at 1732295691114 (+139 ms)Running coprocessor post-close hooks at 1732295691118 (+4 ms)Closed at 1732295691118 2024-11-22T17:14:51,121 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:51,122 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=c70a0885ec0e4f41398ae6890c8dabe5, regionState=CLOSED 2024-11-22T17:14:51,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:14:51,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-22T17:14:51,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure c70a0885ec0e4f41398ae6890c8dabe5, server=e61df7af61d2,42253,1732295655623 in 1.6000 sec 2024-11-22T17:14:51,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-22T17:14:51,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c70a0885ec0e4f41398ae6890c8dabe5, UNASSIGN in 1.6070 sec 2024-11-22T17:14:51,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-22T17:14:51,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6140 sec 2024-11-22T17:14:51,133 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295691133"}]},"ts":"1732295691133"} 2024-11-22T17:14:51,134 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T17:14:51,134 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T17:14:51,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6280 sec 2024-11-22T17:14:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T17:14:51,672 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:51,675 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T17:14:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:14:51,738 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:14:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T17:14:51,742 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:14:51,748 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:51,754 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/recovered.edits] 2024-11-22T17:14:51,757 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/426c234c9111406482ae0a27a48a8956 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/426c234c9111406482ae0a27a48a8956 2024-11-22T17:14:51,758 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/844d3b50778b4a93ba3b7bfd0be23d06 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/844d3b50778b4a93ba3b7bfd0be23d06 2024-11-22T17:14:51,759 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/9aadacaf5936470cbf4ac9e5f0b13c2a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/9aadacaf5936470cbf4ac9e5f0b13c2a 2024-11-22T17:14:51,761 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/a00576187b6d4895b2c26a12ea6932a4 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/A/a00576187b6d4895b2c26a12ea6932a4 2024-11-22T17:14:51,763 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/184ccbd29b914dcbb3c8953433fcacee to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/184ccbd29b914dcbb3c8953433fcacee 2024-11-22T17:14:51,764 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/325154a39dd04cb2a8eb52410b1758f3 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/325154a39dd04cb2a8eb52410b1758f3 2024-11-22T17:14:51,765 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6177020eaabd4f409815e6ec155d6ee8 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/6177020eaabd4f409815e6ec155d6ee8 2024-11-22T17:14:51,767 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/f21d394890f048edab16e04035b43863 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/B/f21d394890f048edab16e04035b43863 2024-11-22T17:14:51,770 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/7bdf67382a3b4445ad2706dec0d95852 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/7bdf67382a3b4445ad2706dec0d95852 2024-11-22T17:14:51,771 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/b48c8505676e4e059ed3b73741f58bb2 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/b48c8505676e4e059ed3b73741f58bb2 2024-11-22T17:14:51,772 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/c108a849f53f4427ae127516db98507d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/c108a849f53f4427ae127516db98507d 2024-11-22T17:14:51,774 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/e5658f6039554b4a87d931dce09ccb0d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/C/e5658f6039554b4a87d931dce09ccb0d 2024-11-22T17:14:51,776 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/recovered.edits/730.seqid to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5/recovered.edits/730.seqid 2024-11-22T17:14:51,777 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/c70a0885ec0e4f41398ae6890c8dabe5 2024-11-22T17:14:51,777 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T17:14:51,781 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:14:51,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-22T17:14:51,791 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T17:14:51,823 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T17:14:51,824 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:14:51,824 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T17:14:51,825 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732295691824"}]},"ts":"9223372036854775807"} 2024-11-22T17:14:51,829 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T17:14:51,829 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c70a0885ec0e4f41398ae6890c8dabe5, NAME => 'TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T17:14:51,830 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T17:14:51,830 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732295691830"}]},"ts":"9223372036854775807"} 2024-11-22T17:14:51,833 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T17:14:51,834 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:14:51,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 158 msec 2024-11-22T17:14:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T17:14:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T17:14:51,852 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:51,862 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_659160475_22 at /127.0.0.1:50140 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;e61df7af61d2:42253-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=328 (was 280) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2153 (was 2739) 2024-11-22T17:14:51,870 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=328, ProcessCount=11, AvailableMemoryMB=2152 2024-11-22T17:14:51,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T17:14:51,872 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T17:14:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T17:14:51,874 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T17:14:51,874 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:51,874 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-22T17:14:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-22T17:14:51,875 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T17:14:51,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741936_1112 (size=1061) 2024-11-22T17:14:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-22T17:14:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-22T17:14:52,286 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8 2024-11-22T17:14:52,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741937_1113 (size=53) 2024-11-22T17:14:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-22T17:14:52,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:14:52,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7e9d66cadc9a648cedc8e55a659274e7, disabling compactions & flushes 2024-11-22T17:14:52,699 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:52,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:52,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. after waiting 0 ms 2024-11-22T17:14:52,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:52,699 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:52,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7e9d66cadc9a648cedc8e55a659274e7: Waiting for close lock at 1732295692698Disabling compacts and flushes for region at 1732295692698Disabling writes for close at 1732295692699 (+1 ms)Writing region close event to WAL at 1732295692699Closed at 1732295692699 2024-11-22T17:14:52,702 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T17:14:52,702 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732295692702"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732295692702"}]},"ts":"1732295692702"} 2024-11-22T17:14:52,705 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T17:14:52,707 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T17:14:52,707 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295692707"}]},"ts":"1732295692707"} 2024-11-22T17:14:52,710 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T17:14:52,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, ASSIGN}] 2024-11-22T17:14:52,712 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, ASSIGN 2024-11-22T17:14:52,713 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, ASSIGN; state=OFFLINE, location=e61df7af61d2,42253,1732295655623; forceNewPlan=false, retain=false 2024-11-22T17:14:52,864 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7e9d66cadc9a648cedc8e55a659274e7, regionState=OPENING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:52,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, ASSIGN because future has completed 2024-11-22T17:14:52,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:14:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-22T17:14:53,026 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:53,026 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} 2024-11-22T17:14:53,027 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:53,027 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:14:53,027 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:53,027 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:53,028 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:53,030 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:14:53,031 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e9d66cadc9a648cedc8e55a659274e7 columnFamilyName A 2024-11-22T17:14:53,031 DEBUG [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:53,032 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.HStore(327): Store=7e9d66cadc9a648cedc8e55a659274e7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:14:53,032 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:53,033 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:14:53,034 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e9d66cadc9a648cedc8e55a659274e7 columnFamilyName B 2024-11-22T17:14:53,034 DEBUG [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:53,034 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.HStore(327): Store=7e9d66cadc9a648cedc8e55a659274e7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:14:53,035 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:53,036 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:14:53,036 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e9d66cadc9a648cedc8e55a659274e7 columnFamilyName C 2024-11-22T17:14:53,036 DEBUG [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:53,037 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.HStore(327): Store=7e9d66cadc9a648cedc8e55a659274e7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:14:53,037 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:53,037 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:53,038 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:53,038 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:53,039 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:53,039 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:53,040 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T17:14:53,041 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:53,043 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T17:14:53,043 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 7e9d66cadc9a648cedc8e55a659274e7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66421436, jitterRate=-0.010243475437164307}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T17:14:53,043 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:53,044 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 7e9d66cadc9a648cedc8e55a659274e7: Running coprocessor pre-open hook at 1732295693027Writing region info on filesystem at 1732295693027Initializing all the Stores at 1732295693028 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295693028Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295693028Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295693028Cleaning up temporary data from old regions at 1732295693039 (+11 ms)Running coprocessor post-open hooks at 1732295693043 (+4 ms)Region opened successfully at 1732295693044 (+1 ms) 2024-11-22T17:14:53,045 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., pid=30, masterSystemTime=1732295693023 2024-11-22T17:14:53,047 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:53,047 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:53,047 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7e9d66cadc9a648cedc8e55a659274e7, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:53,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:14:53,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-22T17:14:53,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 in 182 msec 2024-11-22T17:14:53,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-22T17:14:53,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, ASSIGN in 342 msec 2024-11-22T17:14:53,054 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T17:14:53,055 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295693054"}]},"ts":"1732295693054"} 2024-11-22T17:14:53,056 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T17:14:53,057 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T17:14:53,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-22T17:14:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-22T17:14:54,021 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:54,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f46cc3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:54,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:54,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:54,023 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e641d5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:54,025 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ab66bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:54,026 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:54,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:54,030 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T17:14:54,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is e61df7af61d2,45693,1732295654407 2024-11-22T17:14:54,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3af118e7 2024-11-22T17:14:54,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T17:14:54,032 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T17:14:54,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T17:14:54,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T17:14:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T17:14:54,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741938_1114 (size=1097) 2024-11-22T17:14:54,461 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-22T17:14:54,461 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-22T17:14:54,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T17:14:54,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, REOPEN/MOVE}] 2024-11-22T17:14:54,478 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, REOPEN/MOVE 2024-11-22T17:14:54,479 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7e9d66cadc9a648cedc8e55a659274e7, regionState=CLOSING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:54,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, REOPEN/MOVE because future has completed 2024-11-22T17:14:54,481 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T17:14:54,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:14:54,634 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,635 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T17:14:54,635 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 7e9d66cadc9a648cedc8e55a659274e7, disabling compactions & flushes 2024-11-22T17:14:54,635 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:54,635 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:54,635 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. after waiting 0 ms 2024-11-22T17:14:54,635 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:54,642 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T17:14:54,643 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:54,643 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 7e9d66cadc9a648cedc8e55a659274e7: Waiting for close lock at 1732295694635Running coprocessor pre-close hooks at 1732295694635Disabling compacts and flushes for region at 1732295694635Disabling writes for close at 1732295694635Writing region close event to WAL at 1732295694636 (+1 ms)Running coprocessor post-close hooks at 1732295694643 (+7 ms)Closed at 1732295694643 2024-11-22T17:14:54,644 WARN [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 7e9d66cadc9a648cedc8e55a659274e7 to self. 2024-11-22T17:14:54,647 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,648 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7e9d66cadc9a648cedc8e55a659274e7, regionState=CLOSED 2024-11-22T17:14:54,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:14:54,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-22T17:14:54,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 in 172 msec 2024-11-22T17:14:54,657 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, REOPEN/MOVE; state=CLOSED, location=e61df7af61d2,42253,1732295655623; forceNewPlan=false, retain=true 2024-11-22T17:14:54,757 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T17:14:54,808 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7e9d66cadc9a648cedc8e55a659274e7, regionState=OPENING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:54,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, REOPEN/MOVE because future has completed 2024-11-22T17:14:54,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:14:54,973 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:54,973 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} 2024-11-22T17:14:54,974 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,974 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:14:54,975 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,975 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,977 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,978 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:14:54,982 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e9d66cadc9a648cedc8e55a659274e7 columnFamilyName A 2024-11-22T17:14:54,984 DEBUG [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:54,984 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.HStore(327): Store=7e9d66cadc9a648cedc8e55a659274e7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:14:54,985 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,985 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:14:54,985 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e9d66cadc9a648cedc8e55a659274e7 columnFamilyName B 2024-11-22T17:14:54,985 DEBUG [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:54,986 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.HStore(327): Store=7e9d66cadc9a648cedc8e55a659274e7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:14:54,986 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,987 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:14:54,987 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e9d66cadc9a648cedc8e55a659274e7 columnFamilyName C 2024-11-22T17:14:54,987 DEBUG [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:54,987 INFO [StoreOpener-7e9d66cadc9a648cedc8e55a659274e7-1 {}] regionserver.HStore(327): Store=7e9d66cadc9a648cedc8e55a659274e7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:14:54,987 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:54,987 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,988 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,989 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,990 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,990 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,990 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T17:14:54,992 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,992 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 7e9d66cadc9a648cedc8e55a659274e7; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64149628, jitterRate=-0.04409605264663696}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T17:14:54,992 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:54,993 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 7e9d66cadc9a648cedc8e55a659274e7: Running coprocessor pre-open hook at 1732295694975Writing region info on filesystem at 1732295694975Initializing all the Stores at 1732295694976 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295694976Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295694977 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295694977Cleaning up temporary data from old regions at 1732295694990 (+13 ms)Running coprocessor post-open hooks at 1732295694992 (+2 ms)Region opened successfully at 1732295694993 (+1 ms) 2024-11-22T17:14:54,994 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., pid=35, masterSystemTime=1732295694965 2024-11-22T17:14:54,996 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:54,996 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:54,996 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7e9d66cadc9a648cedc8e55a659274e7, regionState=OPEN, openSeqNum=5, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:54,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:14:55,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-22T17:14:55,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 in 187 msec 2024-11-22T17:14:55,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-22T17:14:55,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, REOPEN/MOVE in 523 msec 2024-11-22T17:14:55,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-22T17:14:55,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 534 msec 2024-11-22T17:14:55,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 967 msec 2024-11-22T17:14:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T17:14:55,011 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:55,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@88c9976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:55,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:55,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ddcfbe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:55,015 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a33850, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa768d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:55,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:55,017 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15684a5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:55,019 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1307436f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc6bab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:55,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:55,021 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292d410f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:55,023 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3752f6c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ddd470f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:55,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:55,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b35760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:55,027 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348bf2eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60093174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:55,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:55,029 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae6ea9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:55,030 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7c4150, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f8203f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:55,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:55,033 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f027ca8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:55,034 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221a5c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6723353c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:55,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:55,036 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f78175f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:55,038 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc1f805, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aece006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:55,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:55,040 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a6e16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:55,041 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e5bca7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe9245f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:14:55,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:14:55,044 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:14:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:14:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:14:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20048632, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:14:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:14:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:14:55,045 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:14:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51dfab01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:14:55,047 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:55,047 DEBUG [Thread-592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:55,047 DEBUG [Thread-588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:55,047 DEBUG [Thread-586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:55,047 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:55,048 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:55,048 DEBUG [Thread-594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:55,049 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:55,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:55,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:55,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:55,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:55,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:55,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:55,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:55,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:55,052 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:14:55,053 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:14:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:14:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:14:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:14:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:14:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:14:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:14:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-22T17:14:55,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:14:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:14:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T17:14:55,061 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:14:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:14:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:14:55,062 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:14:55,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:14:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:14:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:14:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:55,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:14:55,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:14:55,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:55,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:14:55,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:55,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:14:55,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:55,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=759, cellSize=34755, totalHeapSize=79176, min timestamp=1732295695075, max timestamp=1732295695112Num uniques 859; 2024-11-22T17:14:55,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=759, cellSize=34755, totalHeapSize=79176, min timestamp=1732295695075, max timestamp=1732295695112Num uniques 859; 2024-11-22T17:14:55,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=34755, suffix heap size=117064,new segment heap size=79176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=759 2024-11-22T17:14:55,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=34755, suffix heap size=117064,new segment heap size=79176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=759 2024-11-22T17:14:55,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=759, cellSize=34755, totalHeapSize=79176, min timestamp=1732295695075, max timestamp=1732295695112Num uniques 859; 2024-11-22T17:14:55,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=34755, suffix heap size=117064,new segment heap size=79176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=759 2024-11-22T17:14:55,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122dcdc81149fa44bc9973af31d94ae6385_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295695067/Put/seqid=0 2024-11-22T17:14:55,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295755132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295755133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295755135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741939_1115 (size=12154) 2024-11-22T17:14:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295755138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,153 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295755139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,160 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122dcdc81149fa44bc9973af31d94ae6385_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122dcdc81149fa44bc9973af31d94ae6385_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:55,162 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/a15256ba11eb43699f25122d63afd40c, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:55,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/a15256ba11eb43699f25122d63afd40c is 175, key is test_row_0/A:col10/1732295695067/Put/seqid=0 2024-11-22T17:14:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T17:14:55,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741940_1116 (size=30955) 2024-11-22T17:14:55,191 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/a15256ba11eb43699f25122d63afd40c 2024-11-22T17:14:55,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-22T17:14:55,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:55,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:55,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:55,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/9ccbd72e0b164c56b3c8c5d213f213a8 is 50, key is test_row_0/B:col10/1732295695067/Put/seqid=0 2024-11-22T17:14:55,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741941_1117 (size=12001) 2024-11-22T17:14:55,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295755252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295755252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295755252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295755263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295755263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-22T17:14:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:55,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T17:14:55,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295755471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295755472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295755472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295755473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295755473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-22T17:14:55,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:55,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:55,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:55,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/9ccbd72e0b164c56b3c8c5d213f213a8 2024-11-22T17:14:55,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/3b580b3bef994983a242adcb18bb678c is 50, key is test_row_0/C:col10/1732295695067/Put/seqid=0 2024-11-22T17:14:55,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741942_1118 (size=12001) 2024-11-22T17:14:55,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/3b580b3bef994983a242adcb18bb678c 2024-11-22T17:14:55,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-22T17:14:55,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:55,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:55,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:55,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T17:14:55,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/a15256ba11eb43699f25122d63afd40c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a15256ba11eb43699f25122d63afd40c 2024-11-22T17:14:55,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a15256ba11eb43699f25122d63afd40c, entries=150, sequenceid=14, filesize=30.2 K 2024-11-22T17:14:55,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/9ccbd72e0b164c56b3c8c5d213f213a8 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/9ccbd72e0b164c56b3c8c5d213f213a8 2024-11-22T17:14:55,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/9ccbd72e0b164c56b3c8c5d213f213a8, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T17:14:55,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/3b580b3bef994983a242adcb18bb678c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3b580b3bef994983a242adcb18bb678c 2024-11-22T17:14:55,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3b580b3bef994983a242adcb18bb678c, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T17:14:55,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=161.02 KB/164880 for 7e9d66cadc9a648cedc8e55a659274e7 in 653ms, sequenceid=14, compaction requested=false 2024-11-22T17:14:55,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:14:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:55,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=167.72 KB heapSize=428.34 KB 2024-11-22T17:14:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:14:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:14:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:55,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:14:55,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:55,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295755790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295755790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295755795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295755795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295755797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c5a2e6d28df94d308a65999944f65ebd_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295695128/Put/seqid=0 2024-11-22T17:14:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-22T17:14:55,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:55,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:55,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:55,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741943_1119 (size=14594) 2024-11-22T17:14:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,842 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:55,849 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c5a2e6d28df94d308a65999944f65ebd_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c5a2e6d28df94d308a65999944f65ebd_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:55,851 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/13907b13a3ca4104983aecfe0f3af28d, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:55,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/13907b13a3ca4104983aecfe0f3af28d is 175, key is test_row_0/A:col10/1732295695128/Put/seqid=0 2024-11-22T17:14:55,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741944_1120 (size=39549) 2024-11-22T17:14:55,871 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=44, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/13907b13a3ca4104983aecfe0f3af28d 2024-11-22T17:14:55,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/64b84a95b81f48aba2fbbf258321af09 is 50, key is test_row_0/B:col10/1732295695128/Put/seqid=0 2024-11-22T17:14:55,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295755901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295755901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295755912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295755912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295755912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:55,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741945_1121 (size=12001) 2024-11-22T17:14:55,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-22T17:14:55,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:55,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:55,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:55,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295756111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:56,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295756111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:56,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295756121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:56,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295756121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:56,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295756121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:56,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-22T17:14:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:56,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T17:14:56,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-22T17:14:56,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:56,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:56,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:56,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/64b84a95b81f48aba2fbbf258321af09 2024-11-22T17:14:56,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/ee55249c1f8448fcb749bd43866bfe08 is 50, key is test_row_0/C:col10/1732295695128/Put/seqid=0 2024-11-22T17:14:56,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741946_1122 (size=12001) 2024-11-22T17:14:56,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295756421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:56,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295756432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295756432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:56,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295756432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:56,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295756441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:56,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-22T17:14:56,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:56,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:56,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:56,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-22T17:14:56,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:56,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:56,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:56,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,668 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T17:14:56,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-22T17:14:56,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:56,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:56,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:56,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:56,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/ee55249c1f8448fcb749bd43866bfe08 2024-11-22T17:14:56,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/13907b13a3ca4104983aecfe0f3af28d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/13907b13a3ca4104983aecfe0f3af28d 2024-11-22T17:14:56,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/13907b13a3ca4104983aecfe0f3af28d, entries=200, sequenceid=44, filesize=38.6 K 2024-11-22T17:14:56,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/64b84a95b81f48aba2fbbf258321af09 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/64b84a95b81f48aba2fbbf258321af09 2024-11-22T17:14:56,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/64b84a95b81f48aba2fbbf258321af09, entries=150, sequenceid=44, filesize=11.7 K 2024-11-22T17:14:56,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/ee55249c1f8448fcb749bd43866bfe08 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/ee55249c1f8448fcb749bd43866bfe08 2024-11-22T17:14:56,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/ee55249c1f8448fcb749bd43866bfe08, entries=150, sequenceid=44, filesize=11.7 K 2024-11-22T17:14:56,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~428.30 KB/438576, currentSize=33.54 KB/34350 for 7e9d66cadc9a648cedc8e55a659274e7 in 1071ms, sequenceid=44, compaction requested=false 2024-11-22T17:14:56,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:14:56,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-22T17:14:56,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:56,918 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-22T17:14:56,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:14:56,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:56,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:14:56,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:56,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:14:56,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:56,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112262ef2382bfce431d801332b712354cf5_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295695785/Put/seqid=0 2024-11-22T17:14:56,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741947_1123 (size=12154) 2024-11-22T17:14:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:56,945 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112262ef2382bfce431d801332b712354cf5_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112262ef2382bfce431d801332b712354cf5_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:56,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:56,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/79d5039290934fa68336294fd97459eb, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:56,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/79d5039290934fa68336294fd97459eb is 175, key is test_row_0/A:col10/1732295695785/Put/seqid=0 2024-11-22T17:14:56,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741948_1124 (size=30955) 2024-11-22T17:14:56,975 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/79d5039290934fa68336294fd97459eb 2024-11-22T17:14:56,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732295696962, max timestamp=1732295696979Num uniques 859; 2024-11-22T17:14:56,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T17:14:56,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732295696962, max timestamp=1732295696979Num uniques 859; 2024-11-22T17:14:56,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T17:14:56,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/25bad2126a48420482649c23a54128c0 is 50, key is test_row_0/B:col10/1732295695785/Put/seqid=0 2024-11-22T17:14:56,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732295696962, max timestamp=1732295696979Num uniques 859; 2024-11-22T17:14:56,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T17:14:57,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741949_1125 (size=12001) 2024-11-22T17:14:57,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295757020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295757020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295757022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295757025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295757027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295757131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295757131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295757131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295757142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295757142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T17:14:57,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295757343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295757343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295757343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295757352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295757352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,404 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/25bad2126a48420482649c23a54128c0 2024-11-22T17:14:57,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/c1db171e3b7d47fb945aa0968a52cb8a is 50, key is test_row_0/C:col10/1732295695785/Put/seqid=0 2024-11-22T17:14:57,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741950_1126 (size=12001) 2024-11-22T17:14:57,479 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/c1db171e3b7d47fb945aa0968a52cb8a 2024-11-22T17:14:57,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/79d5039290934fa68336294fd97459eb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/79d5039290934fa68336294fd97459eb 2024-11-22T17:14:57,497 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/79d5039290934fa68336294fd97459eb, entries=150, sequenceid=52, filesize=30.2 K 2024-11-22T17:14:57,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/25bad2126a48420482649c23a54128c0 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/25bad2126a48420482649c23a54128c0 2024-11-22T17:14:57,508 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/25bad2126a48420482649c23a54128c0, entries=150, sequenceid=52, filesize=11.7 K 2024-11-22T17:14:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/c1db171e3b7d47fb945aa0968a52cb8a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/c1db171e3b7d47fb945aa0968a52cb8a 2024-11-22T17:14:57,518 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/c1db171e3b7d47fb945aa0968a52cb8a, entries=150, sequenceid=52, filesize=11.7 K 2024-11-22T17:14:57,521 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 7e9d66cadc9a648cedc8e55a659274e7 in 603ms, sequenceid=52, compaction requested=true 2024-11-22T17:14:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:14:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-22T17:14:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-22T17:14:57,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-22T17:14:57,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4610 sec 2024-11-22T17:14:57,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.4740 sec 2024-11-22T17:14:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:57,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=161.02 KB heapSize=445.92 KB 2024-11-22T17:14:57,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:14:57,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:57,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:14:57,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:57,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:14:57,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:14:57,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295757672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295757672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295757677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295757679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295757683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229c5fc8ca212f40ce8911bcc5f1c0b87c_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295697016/Put/seqid=0 2024-11-22T17:14:57,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741951_1127 (size=14594) 2024-11-22T17:14:57,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295757792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295757802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:57,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295757803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295758011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295758022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295758022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,134 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,139 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229c5fc8ca212f40ce8911bcc5f1c0b87c_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229c5fc8ca212f40ce8911bcc5f1c0b87c_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:58,141 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/6d91d8f44e7344f18c6a0cd7a5c5cb86, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:58,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/6d91d8f44e7344f18c6a0cd7a5c5cb86 is 175, key is test_row_0/A:col10/1732295697016/Put/seqid=0 2024-11-22T17:14:58,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741952_1128 (size=39549) 2024-11-22T17:14:58,147 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/6d91d8f44e7344f18c6a0cd7a5c5cb86 2024-11-22T17:14:58,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/c2715a5ec69a4b9fb1f29ad1895dafb1 is 50, key is test_row_0/B:col10/1732295697016/Put/seqid=0 2024-11-22T17:14:58,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741953_1129 (size=12001) 2024-11-22T17:14:58,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/c2715a5ec69a4b9fb1f29ad1895dafb1 2024-11-22T17:14:58,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295758192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295758192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/dcf488afe2f84eadb93188f4618c2cbc is 50, key is test_row_0/C:col10/1732295697016/Put/seqid=0 2024-11-22T17:14:58,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741954_1130 (size=12001) 2024-11-22T17:14:58,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/dcf488afe2f84eadb93188f4618c2cbc 2024-11-22T17:14:58,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/6d91d8f44e7344f18c6a0cd7a5c5cb86 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/6d91d8f44e7344f18c6a0cd7a5c5cb86 2024-11-22T17:14:58,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/6d91d8f44e7344f18c6a0cd7a5c5cb86, entries=200, sequenceid=89, filesize=38.6 K 2024-11-22T17:14:58,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/c2715a5ec69a4b9fb1f29ad1895dafb1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/c2715a5ec69a4b9fb1f29ad1895dafb1 2024-11-22T17:14:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/c2715a5ec69a4b9fb1f29ad1895dafb1, entries=150, sequenceid=89, filesize=11.7 K 2024-11-22T17:14:58,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/dcf488afe2f84eadb93188f4618c2cbc as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dcf488afe2f84eadb93188f4618c2cbc 2024-11-22T17:14:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dcf488afe2f84eadb93188f4618c2cbc, entries=150, sequenceid=89, filesize=11.7 K 2024-11-22T17:14:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.88 KB/456576, currentSize=33.54 KB/34350 for 7e9d66cadc9a648cedc8e55a659274e7 in 617ms, sequenceid=89, compaction requested=true 2024-11-22T17:14:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:14:58,282 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:58,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:14:58,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,283 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:58,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:14:58,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:58,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:14:58,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,284 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:58,284 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/A is initiating minor compaction (all files) 2024-11-22T17:14:58,284 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/A in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:58,285 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a15256ba11eb43699f25122d63afd40c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/13907b13a3ca4104983aecfe0f3af28d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/79d5039290934fa68336294fd97459eb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/6d91d8f44e7344f18c6a0cd7a5c5cb86] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=137.7 K 2024-11-22T17:14:58,285 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,285 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a15256ba11eb43699f25122d63afd40c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/13907b13a3ca4104983aecfe0f3af28d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/79d5039290934fa68336294fd97459eb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/6d91d8f44e7344f18c6a0cd7a5c5cb86] 2024-11-22T17:14:58,285 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:58,285 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/B is initiating minor compaction (all files) 2024-11-22T17:14:58,286 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/B in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:58,286 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/9ccbd72e0b164c56b3c8c5d213f213a8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/64b84a95b81f48aba2fbbf258321af09, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/25bad2126a48420482649c23a54128c0, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/c2715a5ec69a4b9fb1f29ad1895dafb1] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=46.9 K 2024-11-22T17:14:58,286 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting a15256ba11eb43699f25122d63afd40c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732295695057 2024-11-22T17:14:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,286 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ccbd72e0b164c56b3c8c5d213f213a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732295695057 2024-11-22T17:14:58,286 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13907b13a3ca4104983aecfe0f3af28d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732295695128 2024-11-22T17:14:58,287 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 64b84a95b81f48aba2fbbf258321af09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732295695128 2024-11-22T17:14:58,287 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79d5039290934fa68336294fd97459eb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732295695785 2024-11-22T17:14:58,287 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 25bad2126a48420482649c23a54128c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732295695785 2024-11-22T17:14:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,287 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d91d8f44e7344f18c6a0cd7a5c5cb86, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732295697016 2024-11-22T17:14:58,288 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting c2715a5ec69a4b9fb1f29ad1895dafb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732295697016 2024-11-22T17:14:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,313 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#B#compaction#111 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:58,314 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/6357eda9a43b459ab75d4175e1ba0023 is 50, key is test_row_0/B:col10/1732295697016/Put/seqid=0 2024-11-22T17:14:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,323 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,335 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122715026859e394bf28aaefa60a6e742f6_7e9d66cadc9a648cedc8e55a659274e7 store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,340 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122715026859e394bf28aaefa60a6e742f6_7e9d66cadc9a648cedc8e55a659274e7, store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:58,340 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122715026859e394bf28aaefa60a6e742f6_7e9d66cadc9a648cedc8e55a659274e7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741955_1131 (size=12139) 2024-11-22T17:14:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,358 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/6357eda9a43b459ab75d4175e1ba0023 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6357eda9a43b459ab75d4175e1ba0023 2024-11-22T17:14:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:58,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:14:58,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:14:58,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:58,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:14:58,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:58,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:14:58,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741956_1132 (size=4469) 2024-11-22T17:14:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,375 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#A#compaction#112 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,376 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/442a66d9056b4851af5934ffed5741d5 is 175, key is test_row_0/A:col10/1732295697016/Put/seqid=0 2024-11-22T17:14:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,383 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/B of 7e9d66cadc9a648cedc8e55a659274e7 into 6357eda9a43b459ab75d4175e1ba0023(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:58,383 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:14:58,384 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/B, priority=12, startTime=1732295698283; duration=0sec 2024-11-22T17:14:58,384 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:58,384 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:B 2024-11-22T17:14:58,384 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:14:58,386 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:14:58,386 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/C is initiating minor compaction (all files) 2024-11-22T17:14:58,386 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/C in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:58,386 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3b580b3bef994983a242adcb18bb678c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/ee55249c1f8448fcb749bd43866bfe08, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/c1db171e3b7d47fb945aa0968a52cb8a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dcf488afe2f84eadb93188f4618c2cbc] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=46.9 K 2024-11-22T17:14:58,387 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b580b3bef994983a242adcb18bb678c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732295695057 2024-11-22T17:14:58,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741957_1133 (size=31093) 2024-11-22T17:14:58,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122526fac0f2c0d42c8ae317abdde038b3f_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295697676/Put/seqid=0 2024-11-22T17:14:58,389 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting ee55249c1f8448fcb749bd43866bfe08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732295695128 2024-11-22T17:14:58,396 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting c1db171e3b7d47fb945aa0968a52cb8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732295695785 2024-11-22T17:14:58,397 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting dcf488afe2f84eadb93188f4618c2cbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732295697016 2024-11-22T17:14:58,405 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/442a66d9056b4851af5934ffed5741d5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/442a66d9056b4851af5934ffed5741d5 2024-11-22T17:14:58,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741958_1134 (size=14594) 2024-11-22T17:14:58,414 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#C#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:58,415 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/dabd02fab2b84e5d886436aaf5a78674 is 50, key is test_row_0/C:col10/1732295697016/Put/seqid=0 2024-11-22T17:14:58,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741959_1135 (size=12139) 2024-11-22T17:14:58,431 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/A of 7e9d66cadc9a648cedc8e55a659274e7 into 442a66d9056b4851af5934ffed5741d5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:58,431 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:14:58,431 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/A, priority=12, startTime=1732295698282; duration=0sec 2024-11-22T17:14:58,431 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:58,431 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:A 2024-11-22T17:14:58,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295698458, max timestamp=1732295698466Num uniques 859; 2024-11-22T17:14:58,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295698458, max timestamp=1732295698466Num uniques 859; 2024-11-22T17:14:58,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295698458, max timestamp=1732295698466Num uniques 859; 2024-11-22T17:14:58,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:58,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:58,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:14:58,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295758512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295758520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295758522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295758631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295758641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295758641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,814 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:58,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122526fac0f2c0d42c8ae317abdde038b3f_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122526fac0f2c0d42c8ae317abdde038b3f_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:58,824 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/7e18fa3853ec45599727b7e1a26258a5, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:58,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/7e18fa3853ec45599727b7e1a26258a5 is 175, key is test_row_0/A:col10/1732295697676/Put/seqid=0 2024-11-22T17:14:58,834 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/dabd02fab2b84e5d886436aaf5a78674 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dabd02fab2b84e5d886436aaf5a78674 2024-11-22T17:14:58,845 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/C of 7e9d66cadc9a648cedc8e55a659274e7 into dabd02fab2b84e5d886436aaf5a78674(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:58,845 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:14:58,845 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/C, priority=12, startTime=1732295698283; duration=0sec 2024-11-22T17:14:58,845 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:58,845 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:C 2024-11-22T17:14:58,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295758841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741960_1136 (size=39549) 2024-11-22T17:14:58,853 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/7e18fa3853ec45599727b7e1a26258a5 2024-11-22T17:14:58,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295758851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295758851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:58,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/8759849aea684be294f285fedbe5c187 is 50, key is test_row_0/B:col10/1732295697676/Put/seqid=0 2024-11-22T17:14:58,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741961_1137 (size=12001) 2024-11-22T17:14:58,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/8759849aea684be294f285fedbe5c187 2024-11-22T17:14:58,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/124a8b76f5184748b159c6c5163ee553 is 50, key is test_row_0/C:col10/1732295697676/Put/seqid=0 2024-11-22T17:14:58,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741962_1138 (size=12001) 2024-11-22T17:14:58,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/124a8b76f5184748b159c6c5163ee553 2024-11-22T17:14:58,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/7e18fa3853ec45599727b7e1a26258a5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/7e18fa3853ec45599727b7e1a26258a5 2024-11-22T17:14:58,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/7e18fa3853ec45599727b7e1a26258a5, entries=200, sequenceid=99, filesize=38.6 K 2024-11-22T17:14:58,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/8759849aea684be294f285fedbe5c187 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/8759849aea684be294f285fedbe5c187 2024-11-22T17:14:58,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/8759849aea684be294f285fedbe5c187, entries=150, sequenceid=99, filesize=11.7 K 2024-11-22T17:14:58,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/124a8b76f5184748b159c6c5163ee553 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/124a8b76f5184748b159c6c5163ee553 2024-11-22T17:14:58,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/124a8b76f5184748b159c6c5163ee553, entries=150, sequenceid=99, filesize=11.7 K 2024-11-22T17:14:58,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7e9d66cadc9a648cedc8e55a659274e7 in 607ms, sequenceid=99, compaction requested=false 2024-11-22T17:14:58,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:14:59,115 INFO [master/e61df7af61d2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-22T17:14:59,115 INFO [master/e61df7af61d2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-22T17:14:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:59,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-22T17:14:59,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:14:59,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T17:14:59,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:14:59,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T17:14:59,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:14:59,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T17:14:59,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a3968cfef6f44ba195152af1e86c1eef_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295698518/Put/seqid=0 2024-11-22T17:14:59,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295759177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:59,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295759182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:59,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295759183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:59,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741963_1139 (size=14794) 2024-11-22T17:14:59,197 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,204 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a3968cfef6f44ba195152af1e86c1eef_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a3968cfef6f44ba195152af1e86c1eef_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:59,206 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/02442fedb1154599b4a2f4af15bcc290, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:59,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295759201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/02442fedb1154599b4a2f4af15bcc290 is 175, key is test_row_0/A:col10/1732295698518/Put/seqid=0 2024-11-22T17:14:59,207 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:14:59,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:59,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741964_1140 (size=39749) 2024-11-22T17:14:59,214 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/02442fedb1154599b4a2f4af15bcc290 2024-11-22T17:14:59,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295759212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,216 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_0' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:14:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:59,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/0a483a2c9909464a99c65b18374ecd67 is 50, key is test_row_0/B:col10/1732295698518/Put/seqid=0 2024-11-22T17:14:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T17:14:59,231 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:14:59,232 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:14:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-22T17:14:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-22T17:14:59,234 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:14:59,235 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:14:59,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:14:59,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741965_1141 (size=12151) 2024-11-22T17:14:59,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/0a483a2c9909464a99c65b18374ecd67 2024-11-22T17:14:59,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/68f08efb041e4c20b709dc630df42bdc is 50, key is test_row_0/C:col10/1732295698518/Put/seqid=0 2024-11-22T17:14:59,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741966_1142 (size=12151) 2024-11-22T17:14:59,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295759291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295759291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:59,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295759301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-22T17:14:59,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:14:59,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:59,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:59,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:59,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:59,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:59,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295759501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295759501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:59,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295759511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:59,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:14:59,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:59,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:59,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:59,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:59,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-22T17:14:59,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/68f08efb041e4c20b709dc630df42bdc 2024-11-22T17:14:59,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/02442fedb1154599b4a2f4af15bcc290 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/02442fedb1154599b4a2f4af15bcc290 2024-11-22T17:14:59,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/02442fedb1154599b4a2f4af15bcc290, entries=200, sequenceid=140, filesize=38.8 K 2024-11-22T17:14:59,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/0a483a2c9909464a99c65b18374ecd67 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/0a483a2c9909464a99c65b18374ecd67 2024-11-22T17:14:59,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/0a483a2c9909464a99c65b18374ecd67, entries=150, sequenceid=140, filesize=11.9 K 2024-11-22T17:14:59,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/68f08efb041e4c20b709dc630df42bdc as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/68f08efb041e4c20b709dc630df42bdc 2024-11-22T17:14:59,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:14:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:59,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:59,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:59,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:59,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:59,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/68f08efb041e4c20b709dc630df42bdc, entries=150, sequenceid=140, filesize=11.9 K 2024-11-22T17:14:59,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 7e9d66cadc9a648cedc8e55a659274e7 in 536ms, sequenceid=140, compaction requested=true 2024-11-22T17:14:59,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:14:59,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:14:59,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:59,700 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:14:59,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:14:59,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:59,700 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:14:59,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:14:59,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,701 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:14:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,701 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/A is initiating minor compaction (all files) 2024-11-22T17:14:59,701 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/A in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:59,701 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:14:59,702 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/B is initiating minor compaction (all files) 2024-11-22T17:14:59,702 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/442a66d9056b4851af5934ffed5741d5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/7e18fa3853ec45599727b7e1a26258a5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/02442fedb1154599b4a2f4af15bcc290] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=107.8 K 2024-11-22T17:14:59,702 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/B in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:59,702 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:59,702 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6357eda9a43b459ab75d4175e1ba0023, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/8759849aea684be294f285fedbe5c187, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/0a483a2c9909464a99c65b18374ecd67] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=35.4 K 2024-11-22T17:14:59,702 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/442a66d9056b4851af5934ffed5741d5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/7e18fa3853ec45599727b7e1a26258a5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/02442fedb1154599b4a2f4af15bcc290] 2024-11-22T17:14:59,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,702 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 6357eda9a43b459ab75d4175e1ba0023, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732295697016 2024-11-22T17:14:59,702 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 442a66d9056b4851af5934ffed5741d5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732295697016 2024-11-22T17:14:59,703 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e18fa3853ec45599727b7e1a26258a5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732295697674 2024-11-22T17:14:59,703 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 8759849aea684be294f285fedbe5c187, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732295697676 2024-11-22T17:14:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,703 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02442fedb1154599b4a2f4af15bcc290, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732295698507 2024-11-22T17:14:59,703 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a483a2c9909464a99c65b18374ecd67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732295698507 2024-11-22T17:14:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,711 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,714 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#B#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,714 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/77120af169634616ac3950505e19eef4 is 50, key is test_row_0/B:col10/1732295698518/Put/seqid=0 2024-11-22T17:14:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,717 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122e416e74fb47a4fa9a36f0bf11a02bffa_7e9d66cadc9a648cedc8e55a659274e7 store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,720 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122e416e74fb47a4fa9a36f0bf11a02bffa_7e9d66cadc9a648cedc8e55a659274e7, store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:59,720 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e416e74fb47a4fa9a36f0bf11a02bffa_7e9d66cadc9a648cedc8e55a659274e7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:14:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741967_1143 (size=12391) 2024-11-22T17:14:59,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741968_1144 (size=4469) 2024-11-22T17:14:59,770 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#A#compaction#120 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,773 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/2a553dc5eefd449d887d4256aa6f8e7e is 175, key is test_row_0/A:col10/1732295698518/Put/seqid=0 2024-11-22T17:14:59,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,777 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/77120af169634616ac3950505e19eef4 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/77120af169634616ac3950505e19eef4 2024-11-22T17:14:59,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741969_1145 (size=31345) 2024-11-22T17:14:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,785 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/B of 7e9d66cadc9a648cedc8e55a659274e7 into 77120af169634616ac3950505e19eef4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:59,785 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:14:59,785 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/B, priority=13, startTime=1732295699700; duration=0sec 2024-11-22T17:14:59,785 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:14:59,785 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:B 2024-11-22T17:14:59,785 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:14:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,788 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:14:59,788 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/C is initiating minor compaction (all files) 2024-11-22T17:14:59,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,788 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/C in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:59,788 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dabd02fab2b84e5d886436aaf5a78674, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/124a8b76f5184748b159c6c5163ee553, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/68f08efb041e4c20b709dc630df42bdc] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=35.4 K 2024-11-22T17:14:59,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,788 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting dabd02fab2b84e5d886436aaf5a78674, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732295697016 2024-11-22T17:14:59,789 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 124a8b76f5184748b159c6c5163ee553, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732295697676 2024-11-22T17:14:59,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,789 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 68f08efb041e4c20b709dc630df42bdc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732295698507 2024-11-22T17:14:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,813 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#C#compaction#122 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:14:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,814 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/57ca9d0c08ea477f90f629616589fd62 is 50, key is test_row_0/C:col10/1732295698518/Put/seqid=0 2024-11-22T17:14:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:14:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:14:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741970_1146 (size=12391) 2024-11-22T17:14:59,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:14:59,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:59,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:14:59,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:59,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:14:59,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:14:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,846 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/57ca9d0c08ea477f90f629616589fd62 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/57ca9d0c08ea477f90f629616589fd62 2024-11-22T17:14:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:14:59,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:59,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:14:59,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:14:59,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:59,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:14:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a8f71e83f2b5491dbab02a5fe79332ec_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295699825/Put/seqid=0 2024-11-22T17:14:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,857 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/C of 7e9d66cadc9a648cedc8e55a659274e7 into 57ca9d0c08ea477f90f629616589fd62(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:14:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,857 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:14:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,857 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/C, priority=13, startTime=1732295699700; duration=0sec 2024-11-22T17:14:59,857 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:14:59,857 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:C 2024-11-22T17:14:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-22T17:14:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741971_1147 (size=12304) 2024-11-22T17:14:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:14:59,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295759890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295759894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:59,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732295699869, max timestamp=1732295699897Num uniques 859; 2024-11-22T17:14:59,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T17:14:59,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732295699869, max timestamp=1732295699897Num uniques 859; 2024-11-22T17:14:59,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732295699869, max timestamp=1732295699897Num uniques 859; 2024-11-22T17:14:59,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T17:14:59,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T17:14:59,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:14:59,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:14:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295759964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:14:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:14:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:00,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295760011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:00,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295760011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:00,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295760081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:00,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:00,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:00,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,189 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/2a553dc5eefd449d887d4256aa6f8e7e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/2a553dc5eefd449d887d4256aa6f8e7e 2024-11-22T17:15:00,197 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/A of 7e9d66cadc9a648cedc8e55a659274e7 into 2a553dc5eefd449d887d4256aa6f8e7e(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:00,197 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:00,197 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/A, priority=13, startTime=1732295699700; duration=0sec 2024-11-22T17:15:00,197 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:00,197 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:A 2024-11-22T17:15:00,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295760221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:00,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295760223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:00,269 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:00,274 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a8f71e83f2b5491dbab02a5fe79332ec_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a8f71e83f2b5491dbab02a5fe79332ec_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:00,276 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/5605f6cd81b949f5a12a645a6f908d85, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:00,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/5605f6cd81b949f5a12a645a6f908d85 is 175, key is test_row_0/A:col10/1732295699825/Put/seqid=0 2024-11-22T17:15:00,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741972_1148 (size=31101) 2024-11-22T17:15:00,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295760291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:00,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:00,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:00,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-22T17:15:00,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295760531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:00,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295760541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:00,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295760601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:00,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:00,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:00,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,687 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/5605f6cd81b949f5a12a645a6f908d85 2024-11-22T17:15:00,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/fe42213009654207a10e3e09fbd09aef is 50, key is test_row_0/B:col10/1732295699825/Put/seqid=0 2024-11-22T17:15:00,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741973_1149 (size=9757) 2024-11-22T17:15:00,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:00,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:00,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:00,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:00,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295761042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:01,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295761051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:01,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:01,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:01,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295761122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:01,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/fe42213009654207a10e3e09fbd09aef 2024-11-22T17:15:01,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/cb8c94b4e3d44011a88a4c35c5de2c76 is 50, key is test_row_0/C:col10/1732295699825/Put/seqid=0 2024-11-22T17:15:01,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741974_1150 (size=9757) 2024-11-22T17:15:01,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/cb8c94b4e3d44011a88a4c35c5de2c76 2024-11-22T17:15:01,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/5605f6cd81b949f5a12a645a6f908d85 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/5605f6cd81b949f5a12a645a6f908d85 2024-11-22T17:15:01,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/5605f6cd81b949f5a12a645a6f908d85, entries=150, sequenceid=152, filesize=30.4 K 2024-11-22T17:15:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/fe42213009654207a10e3e09fbd09aef as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/fe42213009654207a10e3e09fbd09aef 2024-11-22T17:15:01,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/fe42213009654207a10e3e09fbd09aef, entries=100, sequenceid=152, filesize=9.5 K 2024-11-22T17:15:01,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/cb8c94b4e3d44011a88a4c35c5de2c76 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/cb8c94b4e3d44011a88a4c35c5de2c76 2024-11-22T17:15:01,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/cb8c94b4e3d44011a88a4c35c5de2c76, entries=100, sequenceid=152, filesize=9.5 K 2024-11-22T17:15:01,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 7e9d66cadc9a648cedc8e55a659274e7 in 1338ms, sequenceid=152, compaction requested=false 2024-11-22T17:15:01,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:01,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-22T17:15:01,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:15:01,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:01,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:15:01,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:01,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:15:01,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:01,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f59e4a2854fa4ca698bdd07a84c15f0b_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295699955/Put/seqid=0 2024-11-22T17:15:01,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741975_1151 (size=14794) 2024-11-22T17:15:01,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295761245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:01,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295761246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:01,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295761361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:01,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295761361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-22T17:15:01,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295761571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:01,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:01,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295761581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:01,639 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:01,643 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f59e4a2854fa4ca698bdd07a84c15f0b_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f59e4a2854fa4ca698bdd07a84c15f0b_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:01,644 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/9c4a979287b747a1a5717adf4a94d254, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:01,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/9c4a979287b747a1a5717adf4a94d254 is 175, key is test_row_0/A:col10/1732295699955/Put/seqid=0 2024-11-22T17:15:01,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741976_1152 (size=39749) 2024-11-22T17:15:01,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:01,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:01,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295761881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:01,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:01,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295761891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:01,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:01,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:01,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:01,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,051 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/9c4a979287b747a1a5717adf4a94d254 2024-11-22T17:15:02,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/6d932ea02d9444328b8fccd34e5693f4 is 50, key is test_row_0/B:col10/1732295699955/Put/seqid=0 2024-11-22T17:15:02,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741977_1153 (size=12151) 2024-11-22T17:15:02,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295762061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:02,065 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295762072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:02,075 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:02,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295762141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:02,145 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:02,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:02,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:02,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295762391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:02,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:02,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:02,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:02,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295762401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:02,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:02,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:02,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/6d932ea02d9444328b8fccd34e5693f4 2024-11-22T17:15:02,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/466b911224444457bfb9f1929094e9fa is 50, key is test_row_0/C:col10/1732295699955/Put/seqid=0 2024-11-22T17:15:02,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741978_1154 (size=12151) 2024-11-22T17:15:02,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:02,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:02,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:02,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:02,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:02,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/466b911224444457bfb9f1929094e9fa 2024-11-22T17:15:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/9c4a979287b747a1a5717adf4a94d254 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9c4a979287b747a1a5717adf4a94d254 2024-11-22T17:15:02,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9c4a979287b747a1a5717adf4a94d254, entries=200, sequenceid=189, filesize=38.8 K 2024-11-22T17:15:02,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/6d932ea02d9444328b8fccd34e5693f4 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6d932ea02d9444328b8fccd34e5693f4 2024-11-22T17:15:02,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6d932ea02d9444328b8fccd34e5693f4, entries=150, sequenceid=189, filesize=11.9 K 2024-11-22T17:15:02,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/466b911224444457bfb9f1929094e9fa as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/466b911224444457bfb9f1929094e9fa 2024-11-22T17:15:02,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/466b911224444457bfb9f1929094e9fa, entries=150, sequenceid=189, filesize=11.9 K 2024-11-22T17:15:02,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 7e9d66cadc9a648cedc8e55a659274e7 in 1681ms, sequenceid=189, compaction requested=true 2024-11-22T17:15:02,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:02,908 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:02,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:15:02,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:02,908 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:02,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:15:02,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:02,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:15:02,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:02,909 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:02,909 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/A is initiating minor compaction (all files) 2024-11-22T17:15:02,909 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/A in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,909 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:02,909 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/B is initiating minor compaction (all files) 2024-11-22T17:15:02,910 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/2a553dc5eefd449d887d4256aa6f8e7e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/5605f6cd81b949f5a12a645a6f908d85, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9c4a979287b747a1a5717adf4a94d254] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=99.8 K 2024-11-22T17:15:02,910 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/B in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,910 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,910 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/77120af169634616ac3950505e19eef4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/fe42213009654207a10e3e09fbd09aef, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6d932ea02d9444328b8fccd34e5693f4] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=33.5 K 2024-11-22T17:15:02,910 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/2a553dc5eefd449d887d4256aa6f8e7e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/5605f6cd81b949f5a12a645a6f908d85, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9c4a979287b747a1a5717adf4a94d254] 2024-11-22T17:15:02,910 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 77120af169634616ac3950505e19eef4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732295698507 2024-11-22T17:15:02,910 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a553dc5eefd449d887d4256aa6f8e7e, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732295698507 2024-11-22T17:15:02,910 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting fe42213009654207a10e3e09fbd09aef, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732295699825 2024-11-22T17:15:02,910 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5605f6cd81b949f5a12a645a6f908d85, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732295699180 2024-11-22T17:15:02,911 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d932ea02d9444328b8fccd34e5693f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732295699946 2024-11-22T17:15:02,911 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c4a979287b747a1a5717adf4a94d254, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732295699946 2024-11-22T17:15:02,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-22T17:15:02,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,912 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:15:02,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:15:02,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:02,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:15:02,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:02,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:15:02,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:02,922 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:02,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411221aeae5f8c12343c3949fc850632f7b4e_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295701245/Put/seqid=0 2024-11-22T17:15:02,926 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#B#compaction#131 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:02,926 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/a1618bbd4d3f49a4acadc48889339c1d is 50, key is test_row_0/B:col10/1732295699955/Put/seqid=0 2024-11-22T17:15:02,930 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112203ce3772235a44dbadacc4bd6a55e2f3_7e9d66cadc9a648cedc8e55a659274e7 store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:02,933 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112203ce3772235a44dbadacc4bd6a55e2f3_7e9d66cadc9a648cedc8e55a659274e7, store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:02,933 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112203ce3772235a44dbadacc4bd6a55e2f3_7e9d66cadc9a648cedc8e55a659274e7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:02,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741979_1155 (size=12304) 2024-11-22T17:15:02,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741981_1157 (size=4469) 2024-11-22T17:15:02,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741980_1156 (size=12493) 2024-11-22T17:15:02,957 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/a1618bbd4d3f49a4acadc48889339c1d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/a1618bbd4d3f49a4acadc48889339c1d 2024-11-22T17:15:02,964 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/B of 7e9d66cadc9a648cedc8e55a659274e7 into a1618bbd4d3f49a4acadc48889339c1d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:02,964 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:02,964 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/B, priority=13, startTime=1732295702908; duration=0sec 2024-11-22T17:15:02,964 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:02,964 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:B 2024-11-22T17:15:02,964 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:02,966 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:02,966 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/C is initiating minor compaction (all files) 2024-11-22T17:15:02,966 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/C in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:02,966 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/57ca9d0c08ea477f90f629616589fd62, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/cb8c94b4e3d44011a88a4c35c5de2c76, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/466b911224444457bfb9f1929094e9fa] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=33.5 K 2024-11-22T17:15:02,967 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 57ca9d0c08ea477f90f629616589fd62, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732295698507 2024-11-22T17:15:02,968 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting cb8c94b4e3d44011a88a4c35c5de2c76, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732295699825 2024-11-22T17:15:02,968 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 466b911224444457bfb9f1929094e9fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732295699946 2024-11-22T17:15:02,978 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:02,979 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/dbeea716fece4088b5390f1617a09554 is 50, key is test_row_0/C:col10/1732295699955/Put/seqid=0 2024-11-22T17:15:02,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741982_1158 (size=12493) 2024-11-22T17:15:03,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:03,349 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#A#compaction#129 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:03,349 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/8245edd42dbe46faa369067dfa11c3ca is 175, key is test_row_0/A:col10/1732295699955/Put/seqid=0 2024-11-22T17:15:03,351 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411221aeae5f8c12343c3949fc850632f7b4e_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411221aeae5f8c12343c3949fc850632f7b4e_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:03,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/9d4fb1251d9842ce8c4081ded1013d6c, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:03,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/9d4fb1251d9842ce8c4081ded1013d6c is 175, key is test_row_0/A:col10/1732295701245/Put/seqid=0 2024-11-22T17:15:03,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741983_1159 (size=31447) 2024-11-22T17:15:03,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741984_1160 (size=31105) 2024-11-22T17:15:03,357 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/9d4fb1251d9842ce8c4081ded1013d6c 2024-11-22T17:15:03,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/b767e7cbd0a34ced94f9d4d01b427ca0 is 50, key is test_row_0/B:col10/1732295701245/Put/seqid=0 2024-11-22T17:15:03,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741985_1161 (size=12151) 2024-11-22T17:15:03,369 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/b767e7cbd0a34ced94f9d4d01b427ca0 2024-11-22T17:15:03,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/7ce447583a684dcbb68a339c3db7566d is 50, key is test_row_0/C:col10/1732295701245/Put/seqid=0 2024-11-22T17:15:03,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741986_1162 (size=12151) 2024-11-22T17:15:03,390 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/dbeea716fece4088b5390f1617a09554 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dbeea716fece4088b5390f1617a09554 2024-11-22T17:15:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-22T17:15:03,396 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/C of 7e9d66cadc9a648cedc8e55a659274e7 into dbeea716fece4088b5390f1617a09554(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:03,396 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:03,396 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/C, priority=13, startTime=1732295702909; duration=0sec 2024-11-22T17:15:03,396 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:03,396 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:C 2024-11-22T17:15:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:03,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:03,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732295703442, max timestamp=1732295703455Num uniques 859; 2024-11-22T17:15:03,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732295703442, max timestamp=1732295703455Num uniques 859; 2024-11-22T17:15:03,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T17:15:03,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T17:15:03,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732295703442, max timestamp=1732295703455Num uniques 859; 2024-11-22T17:15:03,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T17:15:03,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295763492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:03,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295763496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:03,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295763601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:03,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295763611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:03,758 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/8245edd42dbe46faa369067dfa11c3ca as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/8245edd42dbe46faa369067dfa11c3ca 2024-11-22T17:15:03,763 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/A of 7e9d66cadc9a648cedc8e55a659274e7 into 8245edd42dbe46faa369067dfa11c3ca(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:03,764 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:03,764 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/A, priority=13, startTime=1732295702908; duration=0sec 2024-11-22T17:15:03,764 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:03,764 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:A 2024-11-22T17:15:03,785 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/7ce447583a684dcbb68a339c3db7566d 2024-11-22T17:15:03,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/9d4fb1251d9842ce8c4081ded1013d6c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9d4fb1251d9842ce8c4081ded1013d6c 2024-11-22T17:15:03,794 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9d4fb1251d9842ce8c4081ded1013d6c, entries=150, sequenceid=199, filesize=30.4 K 2024-11-22T17:15:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/b767e7cbd0a34ced94f9d4d01b427ca0 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b767e7cbd0a34ced94f9d4d01b427ca0 2024-11-22T17:15:03,798 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b767e7cbd0a34ced94f9d4d01b427ca0, entries=150, sequenceid=199, filesize=11.9 K 2024-11-22T17:15:03,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/7ce447583a684dcbb68a339c3db7566d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/7ce447583a684dcbb68a339c3db7566d 2024-11-22T17:15:03,805 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/7ce447583a684dcbb68a339c3db7566d, entries=150, sequenceid=199, filesize=11.9 K 2024-11-22T17:15:03,806 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7e9d66cadc9a648cedc8e55a659274e7 in 894ms, sequenceid=199, compaction requested=false 2024-11-22T17:15:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-22T17:15:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-22T17:15:03,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-22T17:15:03,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.5730 sec 2024-11-22T17:15:03,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 4.5780 sec 2024-11-22T17:15:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:03,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-22T17:15:03,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:15:03,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:03,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:15:03,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:03,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:15:03,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:03,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122943896c274464423af810ab4f3c4bc86_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295703814/Put/seqid=0 2024-11-22T17:15:03,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741987_1163 (size=14794) 2024-11-22T17:15:03,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295763835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:03,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295763836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:03,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295763951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:03,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295763951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:04,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295764072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:04,075 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:04,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:04,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295764081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:04,087 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:04,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295764152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:04,155 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:04,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295764161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:04,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295764161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:04,234 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:04,238 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122943896c274464423af810ab4f3c4bc86_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122943896c274464423af810ab4f3c4bc86_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:04,239 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/9609b3a2f49a40cb88674b117e3a134a, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:04,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/9609b3a2f49a40cb88674b117e3a134a is 175, key is test_row_0/A:col10/1732295703814/Put/seqid=0 2024-11-22T17:15:04,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741988_1164 (size=39749) 2024-11-22T17:15:04,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295764471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295764471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:04,645 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/9609b3a2f49a40cb88674b117e3a134a 2024-11-22T17:15:04,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/b9d461b8062840c6a52cd119f6856f8b is 50, key is test_row_0/B:col10/1732295703814/Put/seqid=0 2024-11-22T17:15:04,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741989_1165 (size=12151) 2024-11-22T17:15:04,757 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T17:15:04,758 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-22T17:15:05,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295765002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:05,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295765002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:05,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/b9d461b8062840c6a52cd119f6856f8b 2024-11-22T17:15:05,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/3854ecfb2c994473855383c398592866 is 50, key is test_row_0/C:col10/1732295703814/Put/seqid=0 2024-11-22T17:15:05,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741990_1166 (size=12151) 2024-11-22T17:15:05,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/3854ecfb2c994473855383c398592866 2024-11-22T17:15:05,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/9609b3a2f49a40cb88674b117e3a134a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9609b3a2f49a40cb88674b117e3a134a 2024-11-22T17:15:05,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9609b3a2f49a40cb88674b117e3a134a, entries=200, sequenceid=238, filesize=38.8 K 2024-11-22T17:15:05,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/b9d461b8062840c6a52cd119f6856f8b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b9d461b8062840c6a52cd119f6856f8b 2024-11-22T17:15:05,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b9d461b8062840c6a52cd119f6856f8b, entries=150, sequenceid=238, filesize=11.9 K 2024-11-22T17:15:05,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/3854ecfb2c994473855383c398592866 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3854ecfb2c994473855383c398592866 2024-11-22T17:15:05,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3854ecfb2c994473855383c398592866, entries=150, sequenceid=238, filesize=11.9 K 2024-11-22T17:15:05,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 7e9d66cadc9a648cedc8e55a659274e7 in 1677ms, sequenceid=238, compaction requested=true 2024-11-22T17:15:05,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:05,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:15:05,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:05,493 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:05,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:15:05,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:05,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:15:05,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:05,493 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:05,494 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:05,494 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:05,494 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/B is initiating minor compaction (all files) 2024-11-22T17:15:05,494 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/A is initiating minor compaction (all files) 2024-11-22T17:15:05,494 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/B in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:05,494 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/A in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:05,494 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/8245edd42dbe46faa369067dfa11c3ca, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9d4fb1251d9842ce8c4081ded1013d6c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9609b3a2f49a40cb88674b117e3a134a] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=99.9 K 2024-11-22T17:15:05,494 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/a1618bbd4d3f49a4acadc48889339c1d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b767e7cbd0a34ced94f9d4d01b427ca0, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b9d461b8062840c6a52cd119f6856f8b] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=35.9 K 2024-11-22T17:15:05,494 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:05,494 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/8245edd42dbe46faa369067dfa11c3ca, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9d4fb1251d9842ce8c4081ded1013d6c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9609b3a2f49a40cb88674b117e3a134a] 2024-11-22T17:15:05,495 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a1618bbd4d3f49a4acadc48889339c1d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732295699946 2024-11-22T17:15:05,496 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8245edd42dbe46faa369067dfa11c3ca, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732295699946 2024-11-22T17:15:05,496 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting b767e7cbd0a34ced94f9d4d01b427ca0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732295701241 2024-11-22T17:15:05,496 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d4fb1251d9842ce8c4081ded1013d6c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732295701241 2024-11-22T17:15:05,496 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting b9d461b8062840c6a52cd119f6856f8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732295703478 2024-11-22T17:15:05,496 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9609b3a2f49a40cb88674b117e3a134a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732295703478 2024-11-22T17:15:05,509 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:05,509 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/a85c2e1544a54f2fba3929f0cf22d92e is 50, key is test_row_0/B:col10/1732295703814/Put/seqid=0 2024-11-22T17:15:05,521 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:05,525 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122a757d959a9f544f6857acfc1cfdad512_7e9d66cadc9a648cedc8e55a659274e7 store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:05,528 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122a757d959a9f544f6857acfc1cfdad512_7e9d66cadc9a648cedc8e55a659274e7, store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:05,528 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a757d959a9f544f6857acfc1cfdad512_7e9d66cadc9a648cedc8e55a659274e7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:05,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741991_1167 (size=12595) 2024-11-22T17:15:05,539 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/a85c2e1544a54f2fba3929f0cf22d92e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/a85c2e1544a54f2fba3929f0cf22d92e 2024-11-22T17:15:05,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741992_1168 (size=4469) 2024-11-22T17:15:05,546 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/B of 7e9d66cadc9a648cedc8e55a659274e7 into a85c2e1544a54f2fba3929f0cf22d92e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:05,546 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:05,546 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/B, priority=13, startTime=1732295705493; duration=0sec 2024-11-22T17:15:05,546 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:05,546 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:B 2024-11-22T17:15:05,547 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:05,548 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:05,548 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/C is initiating minor compaction (all files) 2024-11-22T17:15:05,549 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/C in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:05,549 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dbeea716fece4088b5390f1617a09554, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/7ce447583a684dcbb68a339c3db7566d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3854ecfb2c994473855383c398592866] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=35.9 K 2024-11-22T17:15:05,550 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting dbeea716fece4088b5390f1617a09554, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732295699946 2024-11-22T17:15:05,550 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ce447583a684dcbb68a339c3db7566d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732295701241 2024-11-22T17:15:05,550 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 3854ecfb2c994473855383c398592866, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732295703478 2024-11-22T17:15:05,558 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#C#compaction#140 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:05,559 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/820affefb9d54b12b90205e72b849c96 is 50, key is test_row_0/C:col10/1732295703814/Put/seqid=0 2024-11-22T17:15:05,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741993_1169 (size=12595) 2024-11-22T17:15:05,944 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#A#compaction#139 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:05,944 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/63fde9e7794e4d7c8aca5b717103deeb is 175, key is test_row_0/A:col10/1732295703814/Put/seqid=0 2024-11-22T17:15:05,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741994_1170 (size=31549) 2024-11-22T17:15:05,973 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/820affefb9d54b12b90205e72b849c96 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/820affefb9d54b12b90205e72b849c96 2024-11-22T17:15:05,978 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/C of 7e9d66cadc9a648cedc8e55a659274e7 into 820affefb9d54b12b90205e72b849c96(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:05,978 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:05,978 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/C, priority=13, startTime=1732295705493; duration=0sec 2024-11-22T17:15:05,978 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:05,978 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:C 2024-11-22T17:15:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:06,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:15:06,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:15:06,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:06,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:15:06,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:06,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:15:06,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:06,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d2694b35de114e5e9e6c72ff338a63a2_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295703834/Put/seqid=0 2024-11-22T17:15:06,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741995_1171 (size=14794) 2024-11-22T17:15:06,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732295706041, max timestamp=1732295706055Num uniques 859; 2024-11-22T17:15:06,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T17:15:06,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295706043, max timestamp=1732295706055Num uniques 859; 2024-11-22T17:15:06,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:06,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295706043, max timestamp=1732295706055Num uniques 859; 2024-11-22T17:15:06,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:06,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295766093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:06,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295766094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:06,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295766201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295766201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:06,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:06,354 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/63fde9e7794e4d7c8aca5b717103deeb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/63fde9e7794e4d7c8aca5b717103deeb 2024-11-22T17:15:06,360 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/A of 7e9d66cadc9a648cedc8e55a659274e7 into 63fde9e7794e4d7c8aca5b717103deeb(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:06,360 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:06,361 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/A, priority=13, startTime=1732295705493; duration=0sec 2024-11-22T17:15:06,361 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:06,361 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:A 2024-11-22T17:15:06,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295766411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:06,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295766411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:06,432 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:06,437 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d2694b35de114e5e9e6c72ff338a63a2_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d2694b35de114e5e9e6c72ff338a63a2_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:06,438 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/15766fe32b034972b343f98da0d80a04, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:06,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/15766fe32b034972b343f98da0d80a04 is 175, key is test_row_0/A:col10/1732295703834/Put/seqid=0 2024-11-22T17:15:06,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741996_1172 (size=39749) 2024-11-22T17:15:06,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295766721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:06,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:06,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295766721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:06,842 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/15766fe32b034972b343f98da0d80a04 2024-11-22T17:15:06,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/396274eff361453796813c5506912bdf is 50, key is test_row_0/B:col10/1732295703834/Put/seqid=0 2024-11-22T17:15:06,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741997_1173 (size=12151) 2024-11-22T17:15:07,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295767231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:07,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295767231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:07,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:07,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:07,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:07,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/396274eff361453796813c5506912bdf 2024-11-22T17:15:07,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/9003b99d99484aa7914a3dbb258e73ed is 50, key is test_row_0/C:col10/1732295703834/Put/seqid=0 2024-11-22T17:15:07,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741998_1174 (size=12151) 2024-11-22T17:15:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-22T17:15:07,411 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:07,412 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-22T17:15:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-22T17:15:07,414 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:07,414 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:07,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-22T17:15:07,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-22T17:15:07,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:07,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:07,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:07,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:07,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:07,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-22T17:15:07,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:07,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:07,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:07,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:07,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:07,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/9003b99d99484aa7914a3dbb258e73ed 2024-11-22T17:15:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-22T17:15:07,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/15766fe32b034972b343f98da0d80a04 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/15766fe32b034972b343f98da0d80a04 2024-11-22T17:15:07,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/15766fe32b034972b343f98da0d80a04, entries=200, sequenceid=250, filesize=38.8 K 2024-11-22T17:15:07,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/396274eff361453796813c5506912bdf as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/396274eff361453796813c5506912bdf 2024-11-22T17:15:07,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/396274eff361453796813c5506912bdf, entries=150, sequenceid=250, filesize=11.9 K 2024-11-22T17:15:07,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/9003b99d99484aa7914a3dbb258e73ed as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/9003b99d99484aa7914a3dbb258e73ed 2024-11-22T17:15:07,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/9003b99d99484aa7914a3dbb258e73ed, entries=150, sequenceid=250, filesize=11.9 K 2024-11-22T17:15:07,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7e9d66cadc9a648cedc8e55a659274e7 in 1739ms, sequenceid=250, compaction requested=false 2024-11-22T17:15:07,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:07,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-22T17:15:07,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:07,873 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-22T17:15:07,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:15:07,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:07,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:15:07,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:07,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:15:07,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:07,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226de0843262164f2595536329f407b988_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295706093/Put/seqid=0 2024-11-22T17:15:07,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741999_1175 (size=12454) 2024-11-22T17:15:07,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:07,890 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226de0843262164f2595536329f407b988_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226de0843262164f2595536329f407b988_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:07,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/e223a2f992bb469980728ebe4dc53f30, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/e223a2f992bb469980728ebe4dc53f30 is 175, key is test_row_0/A:col10/1732295706093/Put/seqid=0 2024-11-22T17:15:07,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742000_1176 (size=31255) 2024-11-22T17:15:08,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-22T17:15:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:08,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:08,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295768109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:08,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:08,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295768116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:08,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42002 deadline: 1732295768215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:08,221 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:08,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295768221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:08,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295768231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:08,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295768241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:08,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295768241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:08,245 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,245 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:08,297 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/e223a2f992bb469980728ebe4dc53f30 2024-11-22T17:15:08,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/0152e50537f34fdc9434d3b2c179cc18 is 50, key is test_row_0/B:col10/1732295706093/Put/seqid=0 2024-11-22T17:15:08,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742001_1177 (size=12301) 2024-11-22T17:15:08,310 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/0152e50537f34fdc9434d3b2c179cc18 2024-11-22T17:15:08,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/02b8dd07d609460ea27454af54993534 is 50, key is test_row_0/C:col10/1732295706093/Put/seqid=0 2024-11-22T17:15:08,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742002_1178 (size=12301) 2024-11-22T17:15:08,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295768431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:08,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:08,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295768441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:08,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:08,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-22T17:15:08,723 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/02b8dd07d609460ea27454af54993534 2024-11-22T17:15:08,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/e223a2f992bb469980728ebe4dc53f30 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/e223a2f992bb469980728ebe4dc53f30 2024-11-22T17:15:08,732 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/e223a2f992bb469980728ebe4dc53f30, entries=150, sequenceid=286, filesize=30.5 K 2024-11-22T17:15:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/0152e50537f34fdc9434d3b2c179cc18 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/0152e50537f34fdc9434d3b2c179cc18 2024-11-22T17:15:08,737 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/0152e50537f34fdc9434d3b2c179cc18, entries=150, sequenceid=286, filesize=12.0 K 2024-11-22T17:15:08,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/02b8dd07d609460ea27454af54993534 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/02b8dd07d609460ea27454af54993534 2024-11-22T17:15:08,742 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/02b8dd07d609460ea27454af54993534, entries=150, sequenceid=286, filesize=12.0 K 2024-11-22T17:15:08,743 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 7e9d66cadc9a648cedc8e55a659274e7 in 870ms, sequenceid=286, compaction requested=true 2024-11-22T17:15:08,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:08,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:08,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-22T17:15:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-22T17:15:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:08,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T17:15:08,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:15:08,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:08,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:15:08,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:08,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:15:08,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:08,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-22T17:15:08,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3300 sec 2024-11-22T17:15:08,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.3360 sec 2024-11-22T17:15:08,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220ae0dce8a6dc4aa88bed5d48793df1a6_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295708107/Put/seqid=0 2024-11-22T17:15:08,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742003_1179 (size=14994) 2024-11-22T17:15:08,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:08,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295768798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:08,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:08,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732295708775, max timestamp=1732295708798Num uniques 859; 2024-11-22T17:15:08,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732295708775, max timestamp=1732295708798Num uniques 859; 2024-11-22T17:15:08,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T17:15:08,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T17:15:08,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732295708775, max timestamp=1732295708798Num uniques 859; 2024-11-22T17:15:08,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T17:15:08,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295768856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:08,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:08,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295768901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:08,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295768961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:09,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295769111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:09,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295769171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:09,179 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:09,182 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220ae0dce8a6dc4aa88bed5d48793df1a6_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220ae0dce8a6dc4aa88bed5d48793df1a6_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:09,183 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/337aad3b8b874e4589cec18f4b8bb71e, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:09,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/337aad3b8b874e4589cec18f4b8bb71e is 175, key is test_row_0/A:col10/1732295708107/Put/seqid=0 2024-11-22T17:15:09,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742004_1180 (size=39949) 2024-11-22T17:15:09,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295769421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:09,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295769481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-22T17:15:09,571 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:09,572 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-22T17:15:09,574 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T17:15:09,576 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:09,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:09,593 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/337aad3b8b874e4589cec18f4b8bb71e 2024-11-22T17:15:09,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/b944ad9c4b3b4c1da8d9a6055296fc9c is 50, key is test_row_0/B:col10/1732295708107/Put/seqid=0 2024-11-22T17:15:09,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742005_1181 (size=12301) 2024-11-22T17:15:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T17:15:09,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T17:15:09,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:09,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:09,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:09,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:09,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:09,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T17:15:09,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:09,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:09,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:09,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:09,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T17:15:09,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295769931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:09,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295769992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:10,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/b944ad9c4b3b4c1da8d9a6055296fc9c 2024-11-22T17:15:10,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/395e7f58d72b45af99fefb9f3ea4b899 is 50, key is test_row_0/C:col10/1732295708107/Put/seqid=0 2024-11-22T17:15:10,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742006_1182 (size=12301) 2024-11-22T17:15:10,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T17:15:10,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:10,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:10,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:10,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:10,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:10,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T17:15:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:10,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T17:15:10,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:10,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295770261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295770261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:10,264 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:10,264 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:10,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T17:15:10,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:10,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:10,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:10,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:10,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:10,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/395e7f58d72b45af99fefb9f3ea4b899 2024-11-22T17:15:10,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/337aad3b8b874e4589cec18f4b8bb71e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/337aad3b8b874e4589cec18f4b8bb71e 2024-11-22T17:15:10,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/337aad3b8b874e4589cec18f4b8bb71e, entries=200, sequenceid=297, filesize=39.0 K 2024-11-22T17:15:10,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/b944ad9c4b3b4c1da8d9a6055296fc9c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b944ad9c4b3b4c1da8d9a6055296fc9c 2024-11-22T17:15:10,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b944ad9c4b3b4c1da8d9a6055296fc9c, entries=150, sequenceid=297, filesize=12.0 K 2024-11-22T17:15:10,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/395e7f58d72b45af99fefb9f3ea4b899 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/395e7f58d72b45af99fefb9f3ea4b899 2024-11-22T17:15:10,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/395e7f58d72b45af99fefb9f3ea4b899, entries=150, sequenceid=297, filesize=12.0 K 2024-11-22T17:15:10,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 7e9d66cadc9a648cedc8e55a659274e7 in 1691ms, sequenceid=297, compaction requested=true 2024-11-22T17:15:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:15:10,437 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:15:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:15:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:10,437 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:15:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:15:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:10,439 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142502 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:15:10,439 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/A is initiating minor compaction (all files) 2024-11-22T17:15:10,439 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/A in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:10,439 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/63fde9e7794e4d7c8aca5b717103deeb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/15766fe32b034972b343f98da0d80a04, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/e223a2f992bb469980728ebe4dc53f30, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/337aad3b8b874e4589cec18f4b8bb71e] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=139.2 K 2024-11-22T17:15:10,439 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:10,439 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/63fde9e7794e4d7c8aca5b717103deeb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/15766fe32b034972b343f98da0d80a04, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/e223a2f992bb469980728ebe4dc53f30, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/337aad3b8b874e4589cec18f4b8bb71e] 2024-11-22T17:15:10,439 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:15:10,439 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/B is initiating minor compaction (all files) 2024-11-22T17:15:10,439 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/B in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:10,439 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/a85c2e1544a54f2fba3929f0cf22d92e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/396274eff361453796813c5506912bdf, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/0152e50537f34fdc9434d3b2c179cc18, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b944ad9c4b3b4c1da8d9a6055296fc9c] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=48.2 K 2024-11-22T17:15:10,440 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63fde9e7794e4d7c8aca5b717103deeb, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732295703478 2024-11-22T17:15:10,440 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a85c2e1544a54f2fba3929f0cf22d92e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732295703478 2024-11-22T17:15:10,440 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15766fe32b034972b343f98da0d80a04, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732295703829 2024-11-22T17:15:10,440 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 396274eff361453796813c5506912bdf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732295703829 2024-11-22T17:15:10,441 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting e223a2f992bb469980728ebe4dc53f30, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732295706085 2024-11-22T17:15:10,441 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 0152e50537f34fdc9434d3b2c179cc18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732295706085 2024-11-22T17:15:10,441 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 337aad3b8b874e4589cec18f4b8bb71e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732295708098 2024-11-22T17:15:10,441 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting b944ad9c4b3b4c1da8d9a6055296fc9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732295708098 2024-11-22T17:15:10,450 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#B#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:10,450 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/6e4396531f1849728711cb85ede2a743 is 50, key is test_row_0/B:col10/1732295708107/Put/seqid=0 2024-11-22T17:15:10,452 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:10,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742007_1183 (size=12881) 2024-11-22T17:15:10,457 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122a971003ce49f475c9bec18f406957d99_7e9d66cadc9a648cedc8e55a659274e7 store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:10,459 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122a971003ce49f475c9bec18f406957d99_7e9d66cadc9a648cedc8e55a659274e7, store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:10,459 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a971003ce49f475c9bec18f406957d99_7e9d66cadc9a648cedc8e55a659274e7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:10,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742008_1184 (size=4469) 2024-11-22T17:15:10,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T17:15:10,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:10,494 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-22T17:15:10,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:15:10,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:10,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:15:10,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:10,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:15:10,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:10,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112297aab2cff20b4da9960bad3a6cfa8a63_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295708848/Put/seqid=0 2024-11-22T17:15:10,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742009_1185 (size=12454) 2024-11-22T17:15:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T17:15:10,861 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/6e4396531f1849728711cb85ede2a743 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6e4396531f1849728711cb85ede2a743 2024-11-22T17:15:10,865 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#A#compaction#151 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:10,866 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/B of 7e9d66cadc9a648cedc8e55a659274e7 into 6e4396531f1849728711cb85ede2a743(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:10,866 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:10,866 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/B, priority=12, startTime=1732295710437; duration=0sec 2024-11-22T17:15:10,866 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:10,866 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/b1d2b005534a43dc8a01fac48d2fec71 is 175, key is test_row_0/A:col10/1732295708107/Put/seqid=0 2024-11-22T17:15:10,866 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:B 2024-11-22T17:15:10,866 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:15:10,868 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:15:10,868 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/C is initiating minor compaction (all files) 2024-11-22T17:15:10,868 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/C in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:10,868 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/820affefb9d54b12b90205e72b849c96, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/9003b99d99484aa7914a3dbb258e73ed, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/02b8dd07d609460ea27454af54993534, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/395e7f58d72b45af99fefb9f3ea4b899] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=48.2 K 2024-11-22T17:15:10,869 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 820affefb9d54b12b90205e72b849c96, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732295703478 2024-11-22T17:15:10,869 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 9003b99d99484aa7914a3dbb258e73ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732295703829 2024-11-22T17:15:10,869 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 02b8dd07d609460ea27454af54993534, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732295706085 2024-11-22T17:15:10,870 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 395e7f58d72b45af99fefb9f3ea4b899, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732295708098 2024-11-22T17:15:10,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742010_1186 (size=31835) 2024-11-22T17:15:10,879 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#C#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:10,882 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/b1d2b005534a43dc8a01fac48d2fec71 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/b1d2b005534a43dc8a01fac48d2fec71 2024-11-22T17:15:10,882 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/a6af391f13f648d9a0feadddc9010935 is 50, key is test_row_0/C:col10/1732295708107/Put/seqid=0 2024-11-22T17:15:10,888 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/A of 7e9d66cadc9a648cedc8e55a659274e7 into b1d2b005534a43dc8a01fac48d2fec71(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:10,888 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:10,888 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/A, priority=12, startTime=1732295710437; duration=0sec 2024-11-22T17:15:10,888 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:10,888 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:A 2024-11-22T17:15:10,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742011_1187 (size=12881) 2024-11-22T17:15:10,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:10,915 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112297aab2cff20b4da9960bad3a6cfa8a63_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112297aab2cff20b4da9960bad3a6cfa8a63_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:10,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/22e5b3e3d5554d3aa135ce31bec81ad3, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:10,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/22e5b3e3d5554d3aa135ce31bec81ad3 is 175, key is test_row_0/A:col10/1732295708848/Put/seqid=0 2024-11-22T17:15:10,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742012_1188 (size=31255) 2024-11-22T17:15:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:10,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:10,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295770982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:10,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:10,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:10,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:11,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295771012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:11,013 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_0' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:11,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:11,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:11,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:11,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295771091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:11,303 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/a6af391f13f648d9a0feadddc9010935 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/a6af391f13f648d9a0feadddc9010935 2024-11-22T17:15:11,308 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/C of 7e9d66cadc9a648cedc8e55a659274e7 into a6af391f13f648d9a0feadddc9010935(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:11,308 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:11,308 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/C, priority=12, startTime=1732295710437; duration=0sec 2024-11-22T17:15:11,308 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:11,308 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:C 2024-11-22T17:15:11,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295771311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:11,328 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/22e5b3e3d5554d3aa135ce31bec81ad3 2024-11-22T17:15:11,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/01f24f7872324f73ab4770ca2b8f136c is 50, key is test_row_0/B:col10/1732295708848/Put/seqid=0 2024-11-22T17:15:11,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742013_1189 (size=12301) 2024-11-22T17:15:11,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295771621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T17:15:11,741 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/01f24f7872324f73ab4770ca2b8f136c 2024-11-22T17:15:11,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/3ceb64c0a994404daf2970108b939f13 is 50, key is test_row_0/C:col10/1732295708848/Put/seqid=0 2024-11-22T17:15:11,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742014_1190 (size=12301) 2024-11-22T17:15:12,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295772131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:12,158 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/3ceb64c0a994404daf2970108b939f13 2024-11-22T17:15:12,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/22e5b3e3d5554d3aa135ce31bec81ad3 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/22e5b3e3d5554d3aa135ce31bec81ad3 2024-11-22T17:15:12,224 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/22e5b3e3d5554d3aa135ce31bec81ad3, entries=150, sequenceid=330, filesize=30.5 K 2024-11-22T17:15:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/01f24f7872324f73ab4770ca2b8f136c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/01f24f7872324f73ab4770ca2b8f136c 2024-11-22T17:15:12,229 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/01f24f7872324f73ab4770ca2b8f136c, entries=150, sequenceid=330, filesize=12.0 K 2024-11-22T17:15:12,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/3ceb64c0a994404daf2970108b939f13 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3ceb64c0a994404daf2970108b939f13 2024-11-22T17:15:12,236 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3ceb64c0a994404daf2970108b939f13, entries=150, sequenceid=330, filesize=12.0 K 2024-11-22T17:15:12,240 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for 7e9d66cadc9a648cedc8e55a659274e7 in 1746ms, sequenceid=330, compaction requested=false 2024-11-22T17:15:12,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:12,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:12,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-22T17:15:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-22T17:15:12,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-22T17:15:12,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6650 sec 2024-11-22T17:15:12,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.6710 sec 2024-11-22T17:15:13,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:13,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T17:15:13,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:15:13,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:13,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:15:13,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:13,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:15:13,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:13,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122dba196481f414f72bfeed1408d596711_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295713022/Put/seqid=0 2024-11-22T17:15:13,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742015_1191 (size=14994) 2024-11-22T17:15:13,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:13,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295773096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:13,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42032 deadline: 1732295773152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:13,154 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:13,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295773201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:13,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295773412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:13,434 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:13,438 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122dba196481f414f72bfeed1408d596711_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122dba196481f414f72bfeed1408d596711_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:13,439 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/a6d3b2634fe845c3add73cc783f6ff89, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:13,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/a6d3b2634fe845c3add73cc783f6ff89 is 175, key is test_row_0/A:col10/1732295713022/Put/seqid=0 2024-11-22T17:15:13,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742016_1192 (size=39949) 2024-11-22T17:15:13,515 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T17:15:13,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295773721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T17:15:13,741 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:13,742 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-22T17:15:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T17:15:13,743 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:13,744 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:13,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:13,845 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/a6d3b2634fe845c3add73cc783f6ff89 2024-11-22T17:15:13,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T17:15:13,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/cdb926cdc6e543bea7233d484cb2096c is 50, key is test_row_0/B:col10/1732295713022/Put/seqid=0 2024-11-22T17:15:13,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742017_1193 (size=12301) 2024-11-22T17:15:13,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/cdb926cdc6e543bea7233d484cb2096c 2024-11-22T17:15:13,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/94188d5704ca475a9cba9b3e5ad02ad0 is 50, key is test_row_0/C:col10/1732295713022/Put/seqid=0 2024-11-22T17:15:13,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742018_1194 (size=12301) 2024-11-22T17:15:13,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T17:15:13,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:13,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:13,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:13,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:13,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:14,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T17:15:14,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:14,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:14,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:14,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:14,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T17:15:14,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T17:15:14,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:14,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:14,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:14,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:14,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:14,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42044 deadline: 1732295774231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:14,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42006 deadline: 1732295774272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:14,276 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:14,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42022 deadline: 1732295774281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:14,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/94188d5704ca475a9cba9b3e5ad02ad0 2024-11-22T17:15:14,281 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 7e9d66cadc9a648cedc8e55a659274e7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:14,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:14,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:14,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:14,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/a6d3b2634fe845c3add73cc783f6ff89 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a6d3b2634fe845c3add73cc783f6ff89 2024-11-22T17:15:14,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a6d3b2634fe845c3add73cc783f6ff89, entries=200, sequenceid=345, filesize=39.0 K 2024-11-22T17:15:14,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/cdb926cdc6e543bea7233d484cb2096c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/cdb926cdc6e543bea7233d484cb2096c 2024-11-22T17:15:14,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/cdb926cdc6e543bea7233d484cb2096c, entries=150, sequenceid=345, filesize=12.0 K 2024-11-22T17:15:14,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/94188d5704ca475a9cba9b3e5ad02ad0 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/94188d5704ca475a9cba9b3e5ad02ad0 2024-11-22T17:15:14,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/94188d5704ca475a9cba9b3e5ad02ad0, entries=150, sequenceid=345, filesize=12.0 K 2024-11-22T17:15:14,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7e9d66cadc9a648cedc8e55a659274e7 in 1279ms, sequenceid=345, compaction requested=true 2024-11-22T17:15:14,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:14,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:15:14,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:14,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:15:14,302 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:14,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:14,302 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:14,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e9d66cadc9a648cedc8e55a659274e7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:15:14,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:14,303 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:14,303 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:14,303 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/A is initiating minor compaction (all files) 2024-11-22T17:15:14,303 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/B is initiating minor compaction (all files) 2024-11-22T17:15:14,303 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/A in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:14,303 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/B in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:14,303 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/b1d2b005534a43dc8a01fac48d2fec71, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/22e5b3e3d5554d3aa135ce31bec81ad3, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a6d3b2634fe845c3add73cc783f6ff89] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=100.6 K 2024-11-22T17:15:14,303 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6e4396531f1849728711cb85ede2a743, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/01f24f7872324f73ab4770ca2b8f136c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/cdb926cdc6e543bea7233d484cb2096c] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=36.6 K 2024-11-22T17:15:14,303 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:14,303 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/b1d2b005534a43dc8a01fac48d2fec71, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/22e5b3e3d5554d3aa135ce31bec81ad3, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a6d3b2634fe845c3add73cc783f6ff89] 2024-11-22T17:15:14,304 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1d2b005534a43dc8a01fac48d2fec71, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732295708098 2024-11-22T17:15:14,304 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e4396531f1849728711cb85ede2a743, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732295708098 2024-11-22T17:15:14,304 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22e5b3e3d5554d3aa135ce31bec81ad3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732295708820 2024-11-22T17:15:14,304 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 01f24f7872324f73ab4770ca2b8f136c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732295708820 2024-11-22T17:15:14,304 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6d3b2634fe845c3add73cc783f6ff89, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732295710957 2024-11-22T17:15:14,304 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting cdb926cdc6e543bea7233d484cb2096c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732295710961 2024-11-22T17:15:14,312 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:14,314 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:14,314 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411221c82dcc504e54f3da672c9d18570d2e8_7e9d66cadc9a648cedc8e55a659274e7 store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:14,314 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/d7ca5adc438440e0b2ac6323181aaec1 is 50, key is test_row_0/B:col10/1732295713022/Put/seqid=0 2024-11-22T17:15:14,316 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411221c82dcc504e54f3da672c9d18570d2e8_7e9d66cadc9a648cedc8e55a659274e7, store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:14,316 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411221c82dcc504e54f3da672c9d18570d2e8_7e9d66cadc9a648cedc8e55a659274e7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:14,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742020_1196 (size=4469) 2024-11-22T17:15:14,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742019_1195 (size=12983) 2024-11-22T17:15:14,325 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#A#compaction#160 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:14,325 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/61f7c774b4a94731950abe42fe9f98d1 is 175, key is test_row_0/A:col10/1732295713022/Put/seqid=0 2024-11-22T17:15:14,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742021_1197 (size=31937) 2024-11-22T17:15:14,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T17:15:14,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:14,355 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T17:15:14,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:15:14,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:14,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:15:14,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:14,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:15:14,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:14,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122204e4b36141f4936be9d47e8dab5efcd_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_0/A:col10/1732295713031/Put/seqid=0 2024-11-22T17:15:14,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742022_1198 (size=12454) 2024-11-22T17:15:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T17:15:14,726 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/d7ca5adc438440e0b2ac6323181aaec1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/d7ca5adc438440e0b2ac6323181aaec1 2024-11-22T17:15:14,733 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/B of 7e9d66cadc9a648cedc8e55a659274e7 into d7ca5adc438440e0b2ac6323181aaec1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:14,733 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:14,733 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/B, priority=13, startTime=1732295714302; duration=0sec 2024-11-22T17:15:14,733 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:14,733 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:B 2024-11-22T17:15:14,733 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:14,735 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:14,735 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 7e9d66cadc9a648cedc8e55a659274e7/C is initiating minor compaction (all files) 2024-11-22T17:15:14,736 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e9d66cadc9a648cedc8e55a659274e7/C in TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:14,736 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/a6af391f13f648d9a0feadddc9010935, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3ceb64c0a994404daf2970108b939f13, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/94188d5704ca475a9cba9b3e5ad02ad0] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp, totalSize=36.6 K 2024-11-22T17:15:14,736 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a6af391f13f648d9a0feadddc9010935, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732295708098 2024-11-22T17:15:14,737 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/61f7c774b4a94731950abe42fe9f98d1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/61f7c774b4a94731950abe42fe9f98d1 2024-11-22T17:15:14,737 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ceb64c0a994404daf2970108b939f13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732295708820 2024-11-22T17:15:14,737 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 94188d5704ca475a9cba9b3e5ad02ad0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732295710961 2024-11-22T17:15:14,745 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/A of 7e9d66cadc9a648cedc8e55a659274e7 into 61f7c774b4a94731950abe42fe9f98d1(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:14,745 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:14,745 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/A, priority=13, startTime=1732295714302; duration=0sec 2024-11-22T17:15:14,745 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:14,745 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:A 2024-11-22T17:15:14,749 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e9d66cadc9a648cedc8e55a659274e7#C#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:14,750 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/d4dfaf1692b24b12ac758a43d539db1f is 50, key is test_row_0/C:col10/1732295713022/Put/seqid=0 2024-11-22T17:15:14,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742023_1199 (size=12983) 2024-11-22T17:15:14,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:14,770 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122204e4b36141f4936be9d47e8dab5efcd_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122204e4b36141f4936be9d47e8dab5efcd_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:14,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/1376d377d860471b8a4e9fe4f87a3553, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:14,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/1376d377d860471b8a4e9fe4f87a3553 is 175, key is test_row_0/A:col10/1732295713031/Put/seqid=0 2024-11-22T17:15:14,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742024_1200 (size=31255) 2024-11-22T17:15:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T17:15:15,053 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-11-22T17:15:15,053 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:15,053 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:15,053 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:15,053 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:15,054 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-11-22T17:15:15,054 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:15,054 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:15,054 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:15,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:15,054 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-11-22T17:15:15,055 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:15,055 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:15,055 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:15,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:15,055 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-11-22T17:15:15,055 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:15,055 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:15,055 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:15,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:15,165 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/d4dfaf1692b24b12ac758a43d539db1f as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/d4dfaf1692b24b12ac758a43d539db1f 2024-11-22T17:15:15,172 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e9d66cadc9a648cedc8e55a659274e7/C of 7e9d66cadc9a648cedc8e55a659274e7 into d4dfaf1692b24b12ac758a43d539db1f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:15,172 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:15,172 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., storeName=7e9d66cadc9a648cedc8e55a659274e7/C, priority=13, startTime=1732295714302; duration=0sec 2024-11-22T17:15:15,172 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:15,172 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e9d66cadc9a648cedc8e55a659274e7:C 2024-11-22T17:15:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:15,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. as already flushing 2024-11-22T17:15:15,173 INFO [Thread-592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-592. 2024-11-22T17:15:15,173 DEBUG [Thread-592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:15,173 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:15,173 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:15,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:15,176 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/1376d377d860471b8a4e9fe4f87a3553 2024-11-22T17:15:15,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/8aa8b09e22014ce48f9b8cfdd010fdd5 is 50, key is test_row_0/B:col10/1732295713031/Put/seqid=0 2024-11-22T17:15:15,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742025_1201 (size=12301) 2024-11-22T17:15:15,254 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-11-22T17:15:15,255 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:15,255 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:15,255 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:15,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:15,591 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/8aa8b09e22014ce48f9b8cfdd010fdd5 2024-11-22T17:15:15,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/59e8ec295d424010996d40ae1cd08945 is 50, key is test_row_0/C:col10/1732295713031/Put/seqid=0 2024-11-22T17:15:15,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742026_1202 (size=12301) 2024-11-22T17:15:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T17:15:16,013 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/59e8ec295d424010996d40ae1cd08945 2024-11-22T17:15:16,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/1376d377d860471b8a4e9fe4f87a3553 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/1376d377d860471b8a4e9fe4f87a3553 2024-11-22T17:15:16,031 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/1376d377d860471b8a4e9fe4f87a3553, entries=150, sequenceid=365, filesize=30.5 K 2024-11-22T17:15:16,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/8aa8b09e22014ce48f9b8cfdd010fdd5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/8aa8b09e22014ce48f9b8cfdd010fdd5 2024-11-22T17:15:16,038 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/8aa8b09e22014ce48f9b8cfdd010fdd5, entries=150, sequenceid=365, filesize=12.0 K 2024-11-22T17:15:16,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/59e8ec295d424010996d40ae1cd08945 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/59e8ec295d424010996d40ae1cd08945 2024-11-22T17:15:16,046 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/59e8ec295d424010996d40ae1cd08945, entries=150, sequenceid=365, filesize=12.0 K 2024-11-22T17:15:16,047 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for 7e9d66cadc9a648cedc8e55a659274e7 in 1692ms, sequenceid=365, compaction requested=false 2024-11-22T17:15:16,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 7e9d66cadc9a648cedc8e55a659274e7: 2024-11-22T17:15:16,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:16,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-22T17:15:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-22T17:15:16,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-22T17:15:16,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3050 sec 2024-11-22T17:15:16,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.3100 sec 2024-11-22T17:15:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T17:15:17,901 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:18,233 INFO [Thread-586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-586. 2024-11-22T17:15:18,233 DEBUG [Thread-586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:18,233 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:18,233 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:18,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:24,353 INFO [Thread-594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-594. 2024-11-22T17:15:24,353 DEBUG [Thread-594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:24,353 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:24,353 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:24,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:24,373 INFO [Thread-588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-588. 2024-11-22T17:15:24,373 DEBUG [Thread-588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:24,373 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:24,373 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:24,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2843 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3135 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1233 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3699 rows 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1240 2024-11-22T17:15:24,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3719 rows 2024-11-22T17:15:24,374 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T17:15:24,374 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T17:15:24,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:24,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:24,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:24,375 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T17:15:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T17:15:24,379 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295724379"}]},"ts":"1732295724379"} 2024-11-22T17:15:24,381 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T17:15:24,381 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T17:15:24,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T17:15:24,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, UNASSIGN}] 2024-11-22T17:15:24,385 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, UNASSIGN 2024-11-22T17:15:24,386 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=48 updating hbase:meta row=7e9d66cadc9a648cedc8e55a659274e7, regionState=CLOSING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:24,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, UNASSIGN because future has completed 2024-11-22T17:15:24,388 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T17:15:24,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:15:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T17:15:24,540 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(122): Close 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:24,540 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T17:15:24,540 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1722): Closing 7e9d66cadc9a648cedc8e55a659274e7, disabling compactions & flushes 2024-11-22T17:15:24,540 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:24,540 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:24,541 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. after waiting 0 ms 2024-11-22T17:15:24,541 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:24,541 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(2902): Flushing 7e9d66cadc9a648cedc8e55a659274e7 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-22T17:15:24,541 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=A 2024-11-22T17:15:24,541 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:24,541 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=B 2024-11-22T17:15:24,541 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:24,541 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e9d66cadc9a648cedc8e55a659274e7, store=C 2024-11-22T17:15:24,541 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:24,548 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b5ca19292f29465d9907b58cfa5b617b_7e9d66cadc9a648cedc8e55a659274e7 is 50, key is test_row_1/A:col10/1732295724372/Put/seqid=0 2024-11-22T17:15:24,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742027_1203 (size=9914) 2024-11-22T17:15:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T17:15:24,954 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:24,958 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b5ca19292f29465d9907b58cfa5b617b_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b5ca19292f29465d9907b58cfa5b617b_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:24,959 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/1c4dfe1a9c31438a9d22e85c81d2f71c, store: [table=TestAcidGuarantees family=A region=7e9d66cadc9a648cedc8e55a659274e7] 2024-11-22T17:15:24,960 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/1c4dfe1a9c31438a9d22e85c81d2f71c is 175, key is test_row_1/A:col10/1732295724372/Put/seqid=0 2024-11-22T17:15:24,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742028_1204 (size=22561) 2024-11-22T17:15:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T17:15:25,365 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/1c4dfe1a9c31438a9d22e85c81d2f71c 2024-11-22T17:15:25,374 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/3a024039fd7c4037a31d1c38d0a0e625 is 50, key is test_row_1/B:col10/1732295724372/Put/seqid=0 2024-11-22T17:15:25,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742029_1205 (size=9857) 2024-11-22T17:15:25,384 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/3a024039fd7c4037a31d1c38d0a0e625 2024-11-22T17:15:25,392 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/bb876c58556443df9683066b445404ed is 50, key is test_row_1/C:col10/1732295724372/Put/seqid=0 2024-11-22T17:15:25,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742030_1206 (size=9857) 2024-11-22T17:15:25,397 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/bb876c58556443df9683066b445404ed 2024-11-22T17:15:25,402 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/A/1c4dfe1a9c31438a9d22e85c81d2f71c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/1c4dfe1a9c31438a9d22e85c81d2f71c 2024-11-22T17:15:25,407 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/1c4dfe1a9c31438a9d22e85c81d2f71c, entries=100, sequenceid=376, filesize=22.0 K 2024-11-22T17:15:25,408 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/B/3a024039fd7c4037a31d1c38d0a0e625 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/3a024039fd7c4037a31d1c38d0a0e625 2024-11-22T17:15:25,413 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/3a024039fd7c4037a31d1c38d0a0e625, entries=100, sequenceid=376, filesize=9.6 K 2024-11-22T17:15:25,414 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/.tmp/C/bb876c58556443df9683066b445404ed as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/bb876c58556443df9683066b445404ed 2024-11-22T17:15:25,419 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/bb876c58556443df9683066b445404ed, entries=100, sequenceid=376, filesize=9.6 K 2024-11-22T17:15:25,420 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 7e9d66cadc9a648cedc8e55a659274e7 in 879ms, sequenceid=376, compaction requested=true 2024-11-22T17:15:25,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a15256ba11eb43699f25122d63afd40c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/13907b13a3ca4104983aecfe0f3af28d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/79d5039290934fa68336294fd97459eb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/6d91d8f44e7344f18c6a0cd7a5c5cb86, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/442a66d9056b4851af5934ffed5741d5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/7e18fa3853ec45599727b7e1a26258a5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/02442fedb1154599b4a2f4af15bcc290, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/2a553dc5eefd449d887d4256aa6f8e7e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/5605f6cd81b949f5a12a645a6f908d85, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9c4a979287b747a1a5717adf4a94d254, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/8245edd42dbe46faa369067dfa11c3ca, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9d4fb1251d9842ce8c4081ded1013d6c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9609b3a2f49a40cb88674b117e3a134a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/63fde9e7794e4d7c8aca5b717103deeb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/15766fe32b034972b343f98da0d80a04, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/e223a2f992bb469980728ebe4dc53f30, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/337aad3b8b874e4589cec18f4b8bb71e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/b1d2b005534a43dc8a01fac48d2fec71, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/22e5b3e3d5554d3aa135ce31bec81ad3, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a6d3b2634fe845c3add73cc783f6ff89] to archive 2024-11-22T17:15:25,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:15:25,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a15256ba11eb43699f25122d63afd40c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a15256ba11eb43699f25122d63afd40c 2024-11-22T17:15:25,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/13907b13a3ca4104983aecfe0f3af28d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/13907b13a3ca4104983aecfe0f3af28d 2024-11-22T17:15:25,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/79d5039290934fa68336294fd97459eb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/79d5039290934fa68336294fd97459eb 2024-11-22T17:15:25,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/6d91d8f44e7344f18c6a0cd7a5c5cb86 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/6d91d8f44e7344f18c6a0cd7a5c5cb86 2024-11-22T17:15:25,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/442a66d9056b4851af5934ffed5741d5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/442a66d9056b4851af5934ffed5741d5 2024-11-22T17:15:25,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/7e18fa3853ec45599727b7e1a26258a5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/7e18fa3853ec45599727b7e1a26258a5 2024-11-22T17:15:25,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/02442fedb1154599b4a2f4af15bcc290 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/02442fedb1154599b4a2f4af15bcc290 2024-11-22T17:15:25,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/2a553dc5eefd449d887d4256aa6f8e7e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/2a553dc5eefd449d887d4256aa6f8e7e 2024-11-22T17:15:25,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/5605f6cd81b949f5a12a645a6f908d85 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/5605f6cd81b949f5a12a645a6f908d85 2024-11-22T17:15:25,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9c4a979287b747a1a5717adf4a94d254 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9c4a979287b747a1a5717adf4a94d254 2024-11-22T17:15:25,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/8245edd42dbe46faa369067dfa11c3ca to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/8245edd42dbe46faa369067dfa11c3ca 2024-11-22T17:15:25,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9d4fb1251d9842ce8c4081ded1013d6c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9d4fb1251d9842ce8c4081ded1013d6c 2024-11-22T17:15:25,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9609b3a2f49a40cb88674b117e3a134a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/9609b3a2f49a40cb88674b117e3a134a 2024-11-22T17:15:25,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/63fde9e7794e4d7c8aca5b717103deeb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/63fde9e7794e4d7c8aca5b717103deeb 2024-11-22T17:15:25,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/15766fe32b034972b343f98da0d80a04 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/15766fe32b034972b343f98da0d80a04 2024-11-22T17:15:25,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/e223a2f992bb469980728ebe4dc53f30 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/e223a2f992bb469980728ebe4dc53f30 2024-11-22T17:15:25,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/337aad3b8b874e4589cec18f4b8bb71e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/337aad3b8b874e4589cec18f4b8bb71e 2024-11-22T17:15:25,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/b1d2b005534a43dc8a01fac48d2fec71 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/b1d2b005534a43dc8a01fac48d2fec71 2024-11-22T17:15:25,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/22e5b3e3d5554d3aa135ce31bec81ad3 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/22e5b3e3d5554d3aa135ce31bec81ad3 2024-11-22T17:15:25,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a6d3b2634fe845c3add73cc783f6ff89 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/a6d3b2634fe845c3add73cc783f6ff89 2024-11-22T17:15:25,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/9ccbd72e0b164c56b3c8c5d213f213a8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/64b84a95b81f48aba2fbbf258321af09, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/25bad2126a48420482649c23a54128c0, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6357eda9a43b459ab75d4175e1ba0023, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/c2715a5ec69a4b9fb1f29ad1895dafb1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/8759849aea684be294f285fedbe5c187, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/77120af169634616ac3950505e19eef4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/0a483a2c9909464a99c65b18374ecd67, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/fe42213009654207a10e3e09fbd09aef, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/a1618bbd4d3f49a4acadc48889339c1d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6d932ea02d9444328b8fccd34e5693f4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b767e7cbd0a34ced94f9d4d01b427ca0, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/a85c2e1544a54f2fba3929f0cf22d92e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b9d461b8062840c6a52cd119f6856f8b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/396274eff361453796813c5506912bdf, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/0152e50537f34fdc9434d3b2c179cc18, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6e4396531f1849728711cb85ede2a743, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b944ad9c4b3b4c1da8d9a6055296fc9c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/01f24f7872324f73ab4770ca2b8f136c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/cdb926cdc6e543bea7233d484cb2096c] to archive 2024-11-22T17:15:25,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:15:25,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/9ccbd72e0b164c56b3c8c5d213f213a8 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/9ccbd72e0b164c56b3c8c5d213f213a8 2024-11-22T17:15:25,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/64b84a95b81f48aba2fbbf258321af09 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/64b84a95b81f48aba2fbbf258321af09 2024-11-22T17:15:25,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/25bad2126a48420482649c23a54128c0 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/25bad2126a48420482649c23a54128c0 2024-11-22T17:15:25,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6357eda9a43b459ab75d4175e1ba0023 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6357eda9a43b459ab75d4175e1ba0023 2024-11-22T17:15:25,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/c2715a5ec69a4b9fb1f29ad1895dafb1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/c2715a5ec69a4b9fb1f29ad1895dafb1 2024-11-22T17:15:25,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/8759849aea684be294f285fedbe5c187 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/8759849aea684be294f285fedbe5c187 2024-11-22T17:15:25,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/77120af169634616ac3950505e19eef4 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/77120af169634616ac3950505e19eef4 2024-11-22T17:15:25,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/0a483a2c9909464a99c65b18374ecd67 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/0a483a2c9909464a99c65b18374ecd67 2024-11-22T17:15:25,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/fe42213009654207a10e3e09fbd09aef to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/fe42213009654207a10e3e09fbd09aef 2024-11-22T17:15:25,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/a1618bbd4d3f49a4acadc48889339c1d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/a1618bbd4d3f49a4acadc48889339c1d 2024-11-22T17:15:25,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6d932ea02d9444328b8fccd34e5693f4 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6d932ea02d9444328b8fccd34e5693f4 2024-11-22T17:15:25,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b767e7cbd0a34ced94f9d4d01b427ca0 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b767e7cbd0a34ced94f9d4d01b427ca0 2024-11-22T17:15:25,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/a85c2e1544a54f2fba3929f0cf22d92e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/a85c2e1544a54f2fba3929f0cf22d92e 2024-11-22T17:15:25,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b9d461b8062840c6a52cd119f6856f8b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b9d461b8062840c6a52cd119f6856f8b 2024-11-22T17:15:25,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/396274eff361453796813c5506912bdf to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/396274eff361453796813c5506912bdf 2024-11-22T17:15:25,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/0152e50537f34fdc9434d3b2c179cc18 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/0152e50537f34fdc9434d3b2c179cc18 2024-11-22T17:15:25,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6e4396531f1849728711cb85ede2a743 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/6e4396531f1849728711cb85ede2a743 2024-11-22T17:15:25,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b944ad9c4b3b4c1da8d9a6055296fc9c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/b944ad9c4b3b4c1da8d9a6055296fc9c 2024-11-22T17:15:25,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/01f24f7872324f73ab4770ca2b8f136c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/01f24f7872324f73ab4770ca2b8f136c 2024-11-22T17:15:25,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/cdb926cdc6e543bea7233d484cb2096c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/cdb926cdc6e543bea7233d484cb2096c 2024-11-22T17:15:25,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3b580b3bef994983a242adcb18bb678c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/ee55249c1f8448fcb749bd43866bfe08, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/c1db171e3b7d47fb945aa0968a52cb8a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dabd02fab2b84e5d886436aaf5a78674, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dcf488afe2f84eadb93188f4618c2cbc, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/124a8b76f5184748b159c6c5163ee553, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/57ca9d0c08ea477f90f629616589fd62, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/68f08efb041e4c20b709dc630df42bdc, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/cb8c94b4e3d44011a88a4c35c5de2c76, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dbeea716fece4088b5390f1617a09554, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/466b911224444457bfb9f1929094e9fa, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/7ce447583a684dcbb68a339c3db7566d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/820affefb9d54b12b90205e72b849c96, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3854ecfb2c994473855383c398592866, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/9003b99d99484aa7914a3dbb258e73ed, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/02b8dd07d609460ea27454af54993534, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/a6af391f13f648d9a0feadddc9010935, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/395e7f58d72b45af99fefb9f3ea4b899, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3ceb64c0a994404daf2970108b939f13, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/94188d5704ca475a9cba9b3e5ad02ad0] to archive 2024-11-22T17:15:25,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:15:25,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3b580b3bef994983a242adcb18bb678c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3b580b3bef994983a242adcb18bb678c 2024-11-22T17:15:25,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/ee55249c1f8448fcb749bd43866bfe08 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/ee55249c1f8448fcb749bd43866bfe08 2024-11-22T17:15:25,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/c1db171e3b7d47fb945aa0968a52cb8a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/c1db171e3b7d47fb945aa0968a52cb8a 2024-11-22T17:15:25,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dabd02fab2b84e5d886436aaf5a78674 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dabd02fab2b84e5d886436aaf5a78674 2024-11-22T17:15:25,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dcf488afe2f84eadb93188f4618c2cbc to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dcf488afe2f84eadb93188f4618c2cbc 2024-11-22T17:15:25,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/124a8b76f5184748b159c6c5163ee553 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/124a8b76f5184748b159c6c5163ee553 2024-11-22T17:15:25,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/57ca9d0c08ea477f90f629616589fd62 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/57ca9d0c08ea477f90f629616589fd62 2024-11-22T17:15:25,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/68f08efb041e4c20b709dc630df42bdc to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/68f08efb041e4c20b709dc630df42bdc 2024-11-22T17:15:25,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/cb8c94b4e3d44011a88a4c35c5de2c76 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/cb8c94b4e3d44011a88a4c35c5de2c76 2024-11-22T17:15:25,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dbeea716fece4088b5390f1617a09554 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/dbeea716fece4088b5390f1617a09554 2024-11-22T17:15:25,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/466b911224444457bfb9f1929094e9fa to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/466b911224444457bfb9f1929094e9fa 2024-11-22T17:15:25,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/7ce447583a684dcbb68a339c3db7566d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/7ce447583a684dcbb68a339c3db7566d 2024-11-22T17:15:25,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/820affefb9d54b12b90205e72b849c96 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/820affefb9d54b12b90205e72b849c96 2024-11-22T17:15:25,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3854ecfb2c994473855383c398592866 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3854ecfb2c994473855383c398592866 2024-11-22T17:15:25,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/9003b99d99484aa7914a3dbb258e73ed to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/9003b99d99484aa7914a3dbb258e73ed 2024-11-22T17:15:25,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/02b8dd07d609460ea27454af54993534 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/02b8dd07d609460ea27454af54993534 2024-11-22T17:15:25,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/a6af391f13f648d9a0feadddc9010935 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/a6af391f13f648d9a0feadddc9010935 2024-11-22T17:15:25,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/395e7f58d72b45af99fefb9f3ea4b899 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/395e7f58d72b45af99fefb9f3ea4b899 2024-11-22T17:15:25,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3ceb64c0a994404daf2970108b939f13 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/3ceb64c0a994404daf2970108b939f13 2024-11-22T17:15:25,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/94188d5704ca475a9cba9b3e5ad02ad0 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/94188d5704ca475a9cba9b3e5ad02ad0 2024-11-22T17:15:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T17:15:25,514 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/recovered.edits/379.seqid, newMaxSeqId=379, maxSeqId=4 2024-11-22T17:15:25,515 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. 2024-11-22T17:15:25,515 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1676): Region close journal for 7e9d66cadc9a648cedc8e55a659274e7: Waiting for close lock at 1732295724540Running coprocessor pre-close hooks at 1732295724540Disabling compacts and flushes for region at 1732295724540Disabling writes for close at 1732295724541 (+1 ms)Obtaining lock to block concurrent updates at 1732295724541Preparing flush snapshotting stores in 7e9d66cadc9a648cedc8e55a659274e7 at 1732295724541Finished memstore snapshotting TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732295724541Flushing stores of TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7. at 1732295724542 (+1 ms)Flushing 7e9d66cadc9a648cedc8e55a659274e7/A: creating writer at 1732295724542Flushing mob file 7e9d66cadc9a648cedc8e55a659274e7/A: appending metadata at 1732295724548 (+6 ms)Flushing mob file 7e9d66cadc9a648cedc8e55a659274e7/A: closing flushed file at 1732295724548Flushing 7e9d66cadc9a648cedc8e55a659274e7/A: appending metadata at 1732295724959 (+411 ms)Flushing 7e9d66cadc9a648cedc8e55a659274e7/A: closing flushed file at 1732295724960 (+1 ms)Flushing 7e9d66cadc9a648cedc8e55a659274e7/B: creating writer at 1732295725370 (+410 ms)Flushing 7e9d66cadc9a648cedc8e55a659274e7/B: appending metadata at 1732295725373 (+3 ms)Flushing 7e9d66cadc9a648cedc8e55a659274e7/B: closing flushed file at 1732295725373Flushing 7e9d66cadc9a648cedc8e55a659274e7/C: creating writer at 1732295725389 (+16 ms)Flushing 7e9d66cadc9a648cedc8e55a659274e7/C: appending metadata at 1732295725392 (+3 ms)Flushing 7e9d66cadc9a648cedc8e55a659274e7/C: closing flushed file at 1732295725392Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@152acc21: reopening flushed file at 1732295725401 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51b87800: reopening flushed file at 1732295725407 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b369f5a: reopening flushed file at 1732295725413 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 7e9d66cadc9a648cedc8e55a659274e7 in 879ms, sequenceid=376, compaction requested=true at 1732295725420 (+7 ms)Writing region close event to WAL at 1732295725510 (+90 ms)Running coprocessor post-close hooks at 1732295725514 (+4 ms)Closed at 1732295725514 2024-11-22T17:15:25,517 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(157): Closed 7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:25,517 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=48 updating hbase:meta row=7e9d66cadc9a648cedc8e55a659274e7, regionState=CLOSED 2024-11-22T17:15:25,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=48, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:15:25,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-22T17:15:25,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseRegionProcedure 7e9d66cadc9a648cedc8e55a659274e7, server=e61df7af61d2,42253,1732295655623 in 1.1310 sec 2024-11-22T17:15:25,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-11-22T17:15:25,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e9d66cadc9a648cedc8e55a659274e7, UNASSIGN in 1.1370 sec 2024-11-22T17:15:25,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-22T17:15:25,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1410 sec 2024-11-22T17:15:25,527 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295725526"}]},"ts":"1732295725526"} 2024-11-22T17:15:25,528 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T17:15:25,528 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T17:15:25,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-22T17:15:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T17:15:26,522 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:26,522 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T17:15:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:26,524 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=50, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T17:15:26,525 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=50, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:26,529 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,532 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/recovered.edits] 2024-11-22T17:15:26,536 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/1376d377d860471b8a4e9fe4f87a3553 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/1376d377d860471b8a4e9fe4f87a3553 2024-11-22T17:15:26,537 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/1c4dfe1a9c31438a9d22e85c81d2f71c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/1c4dfe1a9c31438a9d22e85c81d2f71c 2024-11-22T17:15:26,539 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/61f7c774b4a94731950abe42fe9f98d1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/A/61f7c774b4a94731950abe42fe9f98d1 2024-11-22T17:15:26,541 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/3a024039fd7c4037a31d1c38d0a0e625 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/3a024039fd7c4037a31d1c38d0a0e625 2024-11-22T17:15:26,542 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/8aa8b09e22014ce48f9b8cfdd010fdd5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/8aa8b09e22014ce48f9b8cfdd010fdd5 2024-11-22T17:15:26,543 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/d7ca5adc438440e0b2ac6323181aaec1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/B/d7ca5adc438440e0b2ac6323181aaec1 2024-11-22T17:15:26,545 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/59e8ec295d424010996d40ae1cd08945 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/59e8ec295d424010996d40ae1cd08945 2024-11-22T17:15:26,546 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/bb876c58556443df9683066b445404ed to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/bb876c58556443df9683066b445404ed 2024-11-22T17:15:26,547 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/d4dfaf1692b24b12ac758a43d539db1f to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/C/d4dfaf1692b24b12ac758a43d539db1f 2024-11-22T17:15:26,550 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/recovered.edits/379.seqid to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7/recovered.edits/379.seqid 2024-11-22T17:15:26,550 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,551 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T17:15:26,551 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T17:15:26,552 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T17:15:26,555 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220ae0dce8a6dc4aa88bed5d48793df1a6_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220ae0dce8a6dc4aa88bed5d48793df1a6_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,556 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411221aeae5f8c12343c3949fc850632f7b4e_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411221aeae5f8c12343c3949fc850632f7b4e_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,557 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122204e4b36141f4936be9d47e8dab5efcd_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122204e4b36141f4936be9d47e8dab5efcd_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,558 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122526fac0f2c0d42c8ae317abdde038b3f_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122526fac0f2c0d42c8ae317abdde038b3f_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,559 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112262ef2382bfce431d801332b712354cf5_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112262ef2382bfce431d801332b712354cf5_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,560 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226de0843262164f2595536329f407b988_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226de0843262164f2595536329f407b988_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,561 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122943896c274464423af810ab4f3c4bc86_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122943896c274464423af810ab4f3c4bc86_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,562 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112297aab2cff20b4da9960bad3a6cfa8a63_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112297aab2cff20b4da9960bad3a6cfa8a63_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,563 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229c5fc8ca212f40ce8911bcc5f1c0b87c_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229c5fc8ca212f40ce8911bcc5f1c0b87c_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,564 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a3968cfef6f44ba195152af1e86c1eef_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a3968cfef6f44ba195152af1e86c1eef_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,565 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a8f71e83f2b5491dbab02a5fe79332ec_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a8f71e83f2b5491dbab02a5fe79332ec_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,565 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b5ca19292f29465d9907b58cfa5b617b_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b5ca19292f29465d9907b58cfa5b617b_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,566 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c5a2e6d28df94d308a65999944f65ebd_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c5a2e6d28df94d308a65999944f65ebd_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,567 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d2694b35de114e5e9e6c72ff338a63a2_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d2694b35de114e5e9e6c72ff338a63a2_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,568 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122dba196481f414f72bfeed1408d596711_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122dba196481f414f72bfeed1408d596711_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,569 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122dcdc81149fa44bc9973af31d94ae6385_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122dcdc81149fa44bc9973af31d94ae6385_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,570 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f59e4a2854fa4ca698bdd07a84c15f0b_7e9d66cadc9a648cedc8e55a659274e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f59e4a2854fa4ca698bdd07a84c15f0b_7e9d66cadc9a648cedc8e55a659274e7 2024-11-22T17:15:26,570 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T17:15:26,573 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=50, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:26,575 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T17:15:26,577 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T17:15:26,578 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=50, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:26,578 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T17:15:26,578 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732295726578"}]},"ts":"9223372036854775807"} 2024-11-22T17:15:26,581 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T17:15:26,581 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7e9d66cadc9a648cedc8e55a659274e7, NAME => 'TestAcidGuarantees,,1732295691871.7e9d66cadc9a648cedc8e55a659274e7.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T17:15:26,581 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T17:15:26,581 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732295726581"}]},"ts":"9223372036854775807"} 2024-11-22T17:15:26,583 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T17:15:26,584 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=50, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:26,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-22T17:15:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T17:15:26,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T17:15:26,632 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:26,642 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/cluster_c396a244-2610-d741-083f-3e053f26e882/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/cluster_c396a244-2610-d741-083f-3e053f26e882/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_659160475_22 at /127.0.0.1:41824 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=315 (was 328), ProcessCount=11 (was 11), AvailableMemoryMB=3027 (was 2152) - AvailableMemoryMB LEAK? - 2024-11-22T17:15:26,650 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=315, ProcessCount=11, AvailableMemoryMB=3027 2024-11-22T17:15:26,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T17:15:26,651 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T17:15:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:26,653 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T17:15:26,653 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:26,653 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 51 2024-11-22T17:15:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-22T17:15:26,654 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T17:15:26,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742031_1207 (size=1061) 2024-11-22T17:15:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-22T17:15:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-22T17:15:27,063 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8 2024-11-22T17:15:27,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742032_1208 (size=53) 2024-11-22T17:15:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-22T17:15:27,472 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:15:27,472 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0da185830e822d28cb5b7cd0a28bc9f1, disabling compactions & flushes 2024-11-22T17:15:27,472 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:27,472 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:27,472 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. after waiting 0 ms 2024-11-22T17:15:27,472 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:27,472 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:27,472 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0da185830e822d28cb5b7cd0a28bc9f1: Waiting for close lock at 1732295727472Disabling compacts and flushes for region at 1732295727472Disabling writes for close at 1732295727472Writing region close event to WAL at 1732295727472Closed at 1732295727472 2024-11-22T17:15:27,476 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T17:15:27,477 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732295727476"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732295727476"}]},"ts":"1732295727476"} 2024-11-22T17:15:27,480 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T17:15:27,481 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T17:15:27,481 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295727481"}]},"ts":"1732295727481"} 2024-11-22T17:15:27,483 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T17:15:27,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0da185830e822d28cb5b7cd0a28bc9f1, ASSIGN}] 2024-11-22T17:15:27,485 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0da185830e822d28cb5b7cd0a28bc9f1, ASSIGN 2024-11-22T17:15:27,486 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0da185830e822d28cb5b7cd0a28bc9f1, ASSIGN; state=OFFLINE, location=e61df7af61d2,42253,1732295655623; forceNewPlan=false, retain=false 2024-11-22T17:15:27,637 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=0da185830e822d28cb5b7cd0a28bc9f1, regionState=OPENING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:27,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0da185830e822d28cb5b7cd0a28bc9f1, ASSIGN because future has completed 2024-11-22T17:15:27,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:15:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-22T17:15:27,793 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:27,793 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} 2024-11-22T17:15:27,794 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:27,794 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:15:27,794 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:27,794 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:27,795 INFO [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:27,796 INFO [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:15:27,796 INFO [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0da185830e822d28cb5b7cd0a28bc9f1 columnFamilyName A 2024-11-22T17:15:27,796 DEBUG [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:27,797 INFO [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] regionserver.HStore(327): Store=0da185830e822d28cb5b7cd0a28bc9f1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:15:27,797 INFO [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:27,798 INFO [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:15:27,798 INFO [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0da185830e822d28cb5b7cd0a28bc9f1 columnFamilyName B 2024-11-22T17:15:27,798 DEBUG [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:27,799 INFO [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] regionserver.HStore(327): Store=0da185830e822d28cb5b7cd0a28bc9f1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:15:27,799 INFO [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:27,800 INFO [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:15:27,800 INFO [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0da185830e822d28cb5b7cd0a28bc9f1 columnFamilyName C 2024-11-22T17:15:27,800 DEBUG [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:27,800 INFO [StoreOpener-0da185830e822d28cb5b7cd0a28bc9f1-1 {}] regionserver.HStore(327): Store=0da185830e822d28cb5b7cd0a28bc9f1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:15:27,801 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:27,801 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:27,802 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:27,802 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:27,803 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:27,803 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:27,803 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T17:15:27,805 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:27,807 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T17:15:27,807 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 0da185830e822d28cb5b7cd0a28bc9f1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68839982, jitterRate=0.025795668363571167}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T17:15:27,807 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:27,808 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 0da185830e822d28cb5b7cd0a28bc9f1: Running coprocessor pre-open hook at 1732295727794Writing region info on filesystem at 1732295727794Initializing all the Stores at 1732295727795 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295727795Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295727795Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295727795Cleaning up temporary data from old regions at 1732295727803 (+8 ms)Running coprocessor post-open hooks at 1732295727807 (+4 ms)Region opened successfully at 1732295727808 (+1 ms) 2024-11-22T17:15:27,809 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., pid=53, masterSystemTime=1732295727790 2024-11-22T17:15:27,810 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:27,810 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:27,811 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=0da185830e822d28cb5b7cd0a28bc9f1, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:27,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:15:27,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-22T17:15:27,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; OpenRegionProcedure 0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 in 175 msec 2024-11-22T17:15:27,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-22T17:15:27,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0da185830e822d28cb5b7cd0a28bc9f1, ASSIGN in 331 msec 2024-11-22T17:15:27,818 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T17:15:27,818 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295727818"}]},"ts":"1732295727818"} 2024-11-22T17:15:27,819 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T17:15:27,820 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T17:15:27,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-22T17:15:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-22T17:15:28,803 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:28,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78234ef1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:28,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:28,807 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b9371e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:28,810 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3493d69f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,812 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:28,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:28,815 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T17:15:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is e61df7af61d2,45693,1732295654407 2024-11-22T17:15:28,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3b7f4b2b 2024-11-22T17:15:28,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T17:15:28,818 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T17:15:28,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261f8875, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:28,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:28,821 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef463c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:28,823 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:28,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6540f37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443e7dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:28,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:28,826 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:28,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:28,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:28,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7a7a77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:28,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7835371b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa73d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:28,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:28,830 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219155fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:28,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d7fe66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e1cbab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:28,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:28,833 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fa36911, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:28,835 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f7ce77f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a4263d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:28,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:28,836 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae3ea51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:28,838 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:28,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad57e5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ea361c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,839 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:28,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:28,840 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:28,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:28,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:28,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60f58332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:28,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:28,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:28,841 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:28,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ba368f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf3bf24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,842 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:28,842 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:28,843 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc5a0c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:28,844 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11eaca59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e88d13e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,845 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:28,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:28,846 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b18ac3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:28,847 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:28,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e76b18a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62fcffc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:28,848 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:28,849 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec4255a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:28,850 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f152d70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d83ce24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:28,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:28,852 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300bac5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:28,853 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c9e04a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:28,854 DEBUG [Thread-1012 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:28,854 DEBUG [Thread-999 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:28,854 DEBUG [Thread-1004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:28,855 DEBUG [Thread-995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:28,855 DEBUG [Thread-997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:28,855 DEBUG [Thread-1010 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:28,855 DEBUG [Thread-1001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:28,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:28,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:28,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:28,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:28,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:28,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:28,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-22T17:15:28,858 DEBUG [Thread-1006 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:28,858 DEBUG [Thread-1008 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:28,858 DEBUG [Thread-993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T17:15:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:15:28,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:15:28,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:15:28,861 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:15:28,861 DEBUG [Thread-993 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:28,862 DEBUG [Thread-993 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:28,862 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:28,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:28,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:28,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:28,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:15:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:15:28,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:28,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:15:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:15:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:15:28,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:15:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:28,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:15:28,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:28,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:28,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:28,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:28,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:28,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:28,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/bfd21899f715453ea1216b0a7959bd06 is 50, key is test_row_0/A:col10/1732295728870/Put/seqid=0 2024-11-22T17:15:28,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732295728894, max timestamp=1732295728901Num uniques 859; 2024-11-22T17:15:28,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295728894, max timestamp=1732295728901Num uniques 859; 2024-11-22T17:15:28,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:28,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:28,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=188, cellSize=8608, totalHeapSize=19792, min timestamp=1732295728894, max timestamp=1732295728901Num uniques 859; 2024-11-22T17:15:28,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8608, suffix heap size=117064,new segment heap size=19792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=188 2024-11-22T17:15:28,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742033_1209 (size=14337) 2024-11-22T17:15:28,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295788917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:28,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295788921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:28,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:28,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295788922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:28,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295788923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:28,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:28,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295788924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:28,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:28,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:28,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:28,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:28,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T17:15:29,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T17:15:29,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:29,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:29,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:29,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295789031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295789031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295789032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295789032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295789032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T17:15:29,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T17:15:29,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:29,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:29,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295789241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295789242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295789242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295789242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295789242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/bfd21899f715453ea1216b0a7959bd06 2024-11-22T17:15:29,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T17:15:29,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:29,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:29,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:29,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/a18ec83590ac432091ddbbee9dcda0f8 is 50, key is test_row_0/B:col10/1732295728870/Put/seqid=0 2024-11-22T17:15:29,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742034_1210 (size=9657) 2024-11-22T17:15:29,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/a18ec83590ac432091ddbbee9dcda0f8 2024-11-22T17:15:29,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/be28215cbd0449c2898c3268c1fed432 is 50, key is test_row_0/C:col10/1732295728870/Put/seqid=0 2024-11-22T17:15:29,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742035_1211 (size=9657) 2024-11-22T17:15:29,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T17:15:29,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:29,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T17:15:29,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295789551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295789551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295789552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295789552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295789552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:29,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T17:15:29,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:29,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:29,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:29,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/be28215cbd0449c2898c3268c1fed432 2024-11-22T17:15:29,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T17:15:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:29,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:29,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/bfd21899f715453ea1216b0a7959bd06 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bfd21899f715453ea1216b0a7959bd06 2024-11-22T17:15:29,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bfd21899f715453ea1216b0a7959bd06, entries=200, sequenceid=12, filesize=14.0 K 2024-11-22T17:15:29,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/a18ec83590ac432091ddbbee9dcda0f8 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/a18ec83590ac432091ddbbee9dcda0f8 2024-11-22T17:15:29,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/a18ec83590ac432091ddbbee9dcda0f8, entries=100, sequenceid=12, filesize=9.4 K 2024-11-22T17:15:29,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/be28215cbd0449c2898c3268c1fed432 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/be28215cbd0449c2898c3268c1fed432 2024-11-22T17:15:29,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/be28215cbd0449c2898c3268c1fed432, entries=100, sequenceid=12, filesize=9.4 K 2024-11-22T17:15:29,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=133.25 KB/136444 for 0da185830e822d28cb5b7cd0a28bc9f1 in 946ms, sequenceid=12, compaction requested=false 2024-11-22T17:15:29,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:29,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T17:15:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:29,938 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=133.25 KB heapSize=378.16 KB 2024-11-22T17:15:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:29,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/ad9e179afbd6496687b0ea24d2d5114c is 50, key is test_row_0/A:col10/1732295728914/Put/seqid=0 2024-11-22T17:15:29,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742036_1212 (size=12001) 2024-11-22T17:15:29,979 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/ad9e179afbd6496687b0ea24d2d5114c 2024-11-22T17:15:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T17:15:29,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/7d6581e9b2484a2b825f9dc7fa7f52be is 50, key is test_row_0/B:col10/1732295728914/Put/seqid=0 2024-11-22T17:15:30,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742037_1213 (size=12001) 2024-11-22T17:15:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:30,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:30,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295790073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295790074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295790079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295790080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295790080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295790191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295790191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295790191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295790191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295790191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295790401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295790401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295790401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295790401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295790402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,429 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/7d6581e9b2484a2b825f9dc7fa7f52be 2024-11-22T17:15:30,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/2739783baa624217954e61293410c8b9 is 50, key is test_row_0/C:col10/1732295728914/Put/seqid=0 2024-11-22T17:15:30,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742038_1214 (size=12001) 2024-11-22T17:15:30,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295790711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295790711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295790711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295790711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295790711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:30,840 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.79 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/2739783baa624217954e61293410c8b9 2024-11-22T17:15:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/ad9e179afbd6496687b0ea24d2d5114c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/ad9e179afbd6496687b0ea24d2d5114c 2024-11-22T17:15:30,848 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/ad9e179afbd6496687b0ea24d2d5114c, entries=150, sequenceid=48, filesize=11.7 K 2024-11-22T17:15:30,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/7d6581e9b2484a2b825f9dc7fa7f52be as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/7d6581e9b2484a2b825f9dc7fa7f52be 2024-11-22T17:15:30,854 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/7d6581e9b2484a2b825f9dc7fa7f52be, entries=150, sequenceid=48, filesize=11.7 K 2024-11-22T17:15:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/2739783baa624217954e61293410c8b9 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/2739783baa624217954e61293410c8b9 2024-11-22T17:15:30,860 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/2739783baa624217954e61293410c8b9, entries=150, sequenceid=48, filesize=11.7 K 2024-11-22T17:15:30,861 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~133.25 KB/136444, heapSize ~378.12 KB/387192, currentSize=53.67 KB/54960 for 0da185830e822d28cb5b7cd0a28bc9f1 in 923ms, sequenceid=48, compaction requested=false 2024-11-22T17:15:30,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:30,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:30,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-22T17:15:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-22T17:15:30,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-22T17:15:30,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0000 sec 2024-11-22T17:15:30,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.0110 sec 2024-11-22T17:15:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T17:15:31,012 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:31,012 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-22T17:15:31,014 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T17:15:31,015 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:31,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:31,040 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T17:15:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T17:15:31,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-22T17:15:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:31,168 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T17:15:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:31,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:31,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/f98b419bb5f043529178391af022082c is 50, key is test_row_0/A:col10/1732295730078/Put/seqid=0 2024-11-22T17:15:31,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742039_1215 (size=12001) 2024-11-22T17:15:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:31,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:31,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732295731236, max timestamp=1732295731262Num uniques 859; 2024-11-22T17:15:31,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732295731236, max timestamp=1732295731262Num uniques 859; 2024-11-22T17:15:31,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T17:15:31,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T17:15:31,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295731237, max timestamp=1732295731262Num uniques 859; 2024-11-22T17:15:31,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:31,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295791277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295791279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295791285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295791287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295791287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T17:15:31,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295791392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295791392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295791402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295791403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295791403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295791601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295791601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,607 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/f98b419bb5f043529178391af022082c 2024-11-22T17:15:31,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295791612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295791612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295791612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/88caff994b274746979d3ff6993773ab is 50, key is test_row_0/B:col10/1732295730078/Put/seqid=0 2024-11-22T17:15:31,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742040_1216 (size=12001) 2024-11-22T17:15:31,622 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/88caff994b274746979d3ff6993773ab 2024-11-22T17:15:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T17:15:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/754d826347e94adaac13bb6549823acb is 50, key is test_row_0/C:col10/1732295730078/Put/seqid=0 2024-11-22T17:15:31,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742041_1217 (size=12001) 2024-11-22T17:15:31,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295791911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295791911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295791922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295791922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295791922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,065 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/754d826347e94adaac13bb6549823acb 2024-11-22T17:15:32,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/f98b419bb5f043529178391af022082c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f98b419bb5f043529178391af022082c 2024-11-22T17:15:32,073 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f98b419bb5f043529178391af022082c, entries=150, sequenceid=59, filesize=11.7 K 2024-11-22T17:15:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/88caff994b274746979d3ff6993773ab as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/88caff994b274746979d3ff6993773ab 2024-11-22T17:15:32,077 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/88caff994b274746979d3ff6993773ab, entries=150, sequenceid=59, filesize=11.7 K 2024-11-22T17:15:32,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/754d826347e94adaac13bb6549823acb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/754d826347e94adaac13bb6549823acb 2024-11-22T17:15:32,082 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/754d826347e94adaac13bb6549823acb, entries=150, sequenceid=59, filesize=11.7 K 2024-11-22T17:15:32,083 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 0da185830e822d28cb5b7cd0a28bc9f1 in 915ms, sequenceid=59, compaction requested=true 2024-11-22T17:15:32,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:32,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:32,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-22T17:15:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-22T17:15:32,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-22T17:15:32,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0690 sec 2024-11-22T17:15:32,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.0740 sec 2024-11-22T17:15:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T17:15:32,152 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:32,152 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-22T17:15:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T17:15:32,154 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:32,155 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:32,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T17:15:32,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-22T17:15:32,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:32,307 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=129.71 KB heapSize=360.77 KB 2024-11-22T17:15:32,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:32,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:32,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:32,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:32,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:32,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:32,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/de70e72c632d4918acd2986cc98b739b is 50, key is test_row_0/A:col10/1732295731284/Put/seqid=0 2024-11-22T17:15:32,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742042_1218 (size=12001) 2024-11-22T17:15:32,353 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/de70e72c632d4918acd2986cc98b739b 2024-11-22T17:15:32,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/19bb3547648f4e2bbf8673126c77b1eb is 50, key is test_row_0/B:col10/1732295731284/Put/seqid=0 2024-11-22T17:15:32,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742043_1219 (size=12001) 2024-11-22T17:15:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:32,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:32,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295792434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295792435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295792435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295792439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295792438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T17:15:32,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295792552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295792552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295792552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295792552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295792552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295792762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295792762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295792762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,767 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/19bb3547648f4e2bbf8673126c77b1eb 2024-11-22T17:15:32,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295792763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295792763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:32,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:32,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/46d9e5c5304b4a6ebc34a97732eff88a is 50, key is test_row_0/C:col10/1732295731284/Put/seqid=0 2024-11-22T17:15:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T17:15:32,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742044_1220 (size=12001) 2024-11-22T17:15:33,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295793071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295793071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295793071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295793082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295793082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,201 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/46d9e5c5304b4a6ebc34a97732eff88a 2024-11-22T17:15:33,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/de70e72c632d4918acd2986cc98b739b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/de70e72c632d4918acd2986cc98b739b 2024-11-22T17:15:33,210 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/de70e72c632d4918acd2986cc98b739b, entries=150, sequenceid=93, filesize=11.7 K 2024-11-22T17:15:33,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/19bb3547648f4e2bbf8673126c77b1eb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/19bb3547648f4e2bbf8673126c77b1eb 2024-11-22T17:15:33,215 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/19bb3547648f4e2bbf8673126c77b1eb, entries=150, sequenceid=93, filesize=11.7 K 2024-11-22T17:15:33,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/46d9e5c5304b4a6ebc34a97732eff88a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/46d9e5c5304b4a6ebc34a97732eff88a 2024-11-22T17:15:33,221 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/46d9e5c5304b4a6ebc34a97732eff88a, entries=150, sequenceid=93, filesize=11.7 K 2024-11-22T17:15:33,222 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~360.72 KB/369376, currentSize=53.67 KB/54960 for 0da185830e822d28cb5b7cd0a28bc9f1 in 915ms, sequenceid=93, compaction requested=true 2024-11-22T17:15:33,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:33,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:33,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-22T17:15:33,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-22T17:15:33,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-22T17:15:33,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0680 sec 2024-11-22T17:15:33,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.0740 sec 2024-11-22T17:15:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T17:15:33,292 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:33,292 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-22T17:15:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T17:15:33,294 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:33,295 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:33,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T17:15:33,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-22T17:15:33,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:33,448 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T17:15:33,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:33,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:33,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:33,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:33,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:33,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:33,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/99e2ecaa39d14e6c896ca8c01fae9f84 is 50, key is test_row_0/A:col10/1732295732436/Put/seqid=0 2024-11-22T17:15:33,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742045_1221 (size=9657) 2024-11-22T17:15:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:33,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T17:15:33,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295793605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295733603, max timestamp=1732295733610Num uniques 859; 2024-11-22T17:15:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295793606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295733603, max timestamp=1732295733610Num uniques 859; 2024-11-22T17:15:33,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:33,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:33,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295733603, max timestamp=1732295733610Num uniques 859; 2024-11-22T17:15:33,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295793635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295793638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295793639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295793721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295793722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295793751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295793751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295793751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,858 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/99e2ecaa39d14e6c896ca8c01fae9f84 2024-11-22T17:15:33,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/9e158601273e414ca1cf124e838539fa is 50, key is test_row_0/B:col10/1732295732436/Put/seqid=0 2024-11-22T17:15:33,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742046_1222 (size=9657) 2024-11-22T17:15:33,870 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/9e158601273e414ca1cf124e838539fa 2024-11-22T17:15:33,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/068cba49975d4ffbb94ec36f4f95151a is 50, key is test_row_0/C:col10/1732295732436/Put/seqid=0 2024-11-22T17:15:33,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742047_1223 (size=9657) 2024-11-22T17:15:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T17:15:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295793931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295793931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295793961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295793961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295793961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:34,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:34,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295794242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:34,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:34,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295794242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:34,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:34,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:34,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:34,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295794271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:34,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:34,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295794271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:34,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295794271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:34,282 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/068cba49975d4ffbb94ec36f4f95151a 2024-11-22T17:15:34,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/99e2ecaa39d14e6c896ca8c01fae9f84 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/99e2ecaa39d14e6c896ca8c01fae9f84 2024-11-22T17:15:34,290 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/99e2ecaa39d14e6c896ca8c01fae9f84, entries=100, sequenceid=104, filesize=9.4 K 2024-11-22T17:15:34,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/9e158601273e414ca1cf124e838539fa as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/9e158601273e414ca1cf124e838539fa 2024-11-22T17:15:34,295 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/9e158601273e414ca1cf124e838539fa, entries=100, sequenceid=104, filesize=9.4 K 2024-11-22T17:15:34,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/068cba49975d4ffbb94ec36f4f95151a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/068cba49975d4ffbb94ec36f4f95151a 2024-11-22T17:15:34,304 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/068cba49975d4ffbb94ec36f4f95151a, entries=100, sequenceid=104, filesize=9.4 K 2024-11-22T17:15:34,305 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0da185830e822d28cb5b7cd0a28bc9f1 in 857ms, sequenceid=104, compaction requested=true 2024-11-22T17:15:34,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:34,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:34,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-22T17:15:34,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-22T17:15:34,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-22T17:15:34,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0110 sec 2024-11-22T17:15:34,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.0170 sec 2024-11-22T17:15:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T17:15:34,432 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:34,432 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-22T17:15:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T17:15:34,434 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:34,435 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:34,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T17:15:34,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T17:15:34,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:34,587 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-22T17:15:34,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:34,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/45d0ab5c96ed4571847b0005c20af773 is 50, key is test_row_0/A:col10/1732295733637/Put/seqid=0 2024-11-22T17:15:34,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742048_1224 (size=12151) 2024-11-22T17:15:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T17:15:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:34,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:34,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295794780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:34,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295794781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:34,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295794781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:34,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295794781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295794783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:34,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295794891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:34,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:34,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295794902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:35,000 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/45d0ab5c96ed4571847b0005c20af773 2024-11-22T17:15:35,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/83f7f61741914d3ab9889957963951af is 50, key is test_row_0/B:col10/1732295733637/Put/seqid=0 2024-11-22T17:15:35,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742049_1225 (size=12151) 2024-11-22T17:15:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T17:15:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295795101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:35,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295795111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:35,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:35,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295795411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:35,414 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/83f7f61741914d3ab9889957963951af 2024-11-22T17:15:35,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/e96ee8c497ee40c2936b5990a5e377ef is 50, key is test_row_0/C:col10/1732295733637/Put/seqid=0 2024-11-22T17:15:35,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742050_1226 (size=12151) 2024-11-22T17:15:35,426 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/e96ee8c497ee40c2936b5990a5e377ef 2024-11-22T17:15:35,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295795421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:35,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/45d0ab5c96ed4571847b0005c20af773 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/45d0ab5c96ed4571847b0005c20af773 2024-11-22T17:15:35,436 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/45d0ab5c96ed4571847b0005c20af773, entries=150, sequenceid=139, filesize=11.9 K 2024-11-22T17:15:35,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/83f7f61741914d3ab9889957963951af as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/83f7f61741914d3ab9889957963951af 2024-11-22T17:15:35,442 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/83f7f61741914d3ab9889957963951af, entries=150, sequenceid=139, filesize=11.9 K 2024-11-22T17:15:35,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/e96ee8c497ee40c2936b5990a5e377ef as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/e96ee8c497ee40c2936b5990a5e377ef 2024-11-22T17:15:35,447 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/e96ee8c497ee40c2936b5990a5e377ef, entries=150, sequenceid=139, filesize=11.9 K 2024-11-22T17:15:35,448 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 0da185830e822d28cb5b7cd0a28bc9f1 in 861ms, sequenceid=139, compaction requested=true 2024-11-22T17:15:35,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:35,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:35,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-22T17:15:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-22T17:15:35,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-22T17:15:35,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0150 sec 2024-11-22T17:15:35,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.0200 sec 2024-11-22T17:15:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T17:15:35,572 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:35,572 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-22T17:15:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T17:15:35,574 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:35,575 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:35,575 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T17:15:35,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-22T17:15:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:35,727 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T17:15:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:35,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/e3650967e2c1418a99dc2134684215ca is 50, key is test_row_0/A:col10/1732295734781/Put/seqid=0 2024-11-22T17:15:35,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742051_1227 (size=12151) 2024-11-22T17:15:35,737 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/e3650967e2c1418a99dc2134684215ca 2024-11-22T17:15:35,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/3b0e2b3deaf741e1b9800c2ef106d275 is 50, key is test_row_0/B:col10/1732295734781/Put/seqid=0 2024-11-22T17:15:35,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742052_1228 (size=12151) 2024-11-22T17:15:35,749 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/3b0e2b3deaf741e1b9800c2ef106d275 2024-11-22T17:15:35,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/5ca3a8c4520344398510a8ef11b8e54e is 50, key is test_row_0/C:col10/1732295734781/Put/seqid=0 2024-11-22T17:15:35,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742053_1229 (size=12151) 2024-11-22T17:15:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:35,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:35,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732295735839, max timestamp=1732295735852Num uniques 859; 2024-11-22T17:15:35,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732295735839, max timestamp=1732295735852Num uniques 859; 2024-11-22T17:15:35,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732295735839, max timestamp=1732295735852Num uniques 859; 2024-11-22T17:15:35,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T17:15:35,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T17:15:35,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T17:15:35,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295795882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T17:15:35,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295795886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295795887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:35,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295795921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:35,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295795941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:35,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295795991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295796001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295796001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,164 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/5ca3a8c4520344398510a8ef11b8e54e 2024-11-22T17:15:36,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/e3650967e2c1418a99dc2134684215ca as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e3650967e2c1418a99dc2134684215ca 2024-11-22T17:15:36,173 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e3650967e2c1418a99dc2134684215ca, entries=150, sequenceid=150, filesize=11.9 K 2024-11-22T17:15:36,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/3b0e2b3deaf741e1b9800c2ef106d275 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/3b0e2b3deaf741e1b9800c2ef106d275 2024-11-22T17:15:36,178 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/3b0e2b3deaf741e1b9800c2ef106d275, entries=150, sequenceid=150, filesize=11.9 K 2024-11-22T17:15:36,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/5ca3a8c4520344398510a8ef11b8e54e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/5ca3a8c4520344398510a8ef11b8e54e 2024-11-22T17:15:36,183 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/5ca3a8c4520344398510a8ef11b8e54e, entries=150, sequenceid=150, filesize=11.9 K 2024-11-22T17:15:36,183 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0da185830e822d28cb5b7cd0a28bc9f1 in 456ms, sequenceid=150, compaction requested=true 2024-11-22T17:15:36,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:36,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-22T17:15:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-22T17:15:36,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-22T17:15:36,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 610 msec 2024-11-22T17:15:36,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 615 msec 2024-11-22T17:15:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T17:15:36,202 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:36,202 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-22T17:15:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T17:15:36,204 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:36,205 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:36,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:36,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-22T17:15:36,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:36,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:36,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:36,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:36,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:36,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:36,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/e8fb01058efb47268ae389ad0904b5c7 is 50, key is test_row_0/A:col10/1732295736206/Put/seqid=0 2024-11-22T17:15:36,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742054_1230 (size=14541) 2024-11-22T17:15:36,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/e8fb01058efb47268ae389ad0904b5c7 2024-11-22T17:15:36,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/3d53b9a0b48648288af7861586844a42 is 50, key is test_row_0/B:col10/1732295736206/Put/seqid=0 2024-11-22T17:15:36,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295796227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295796227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295796232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742055_1231 (size=12151) 2024-11-22T17:15:36,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/3d53b9a0b48648288af7861586844a42 2024-11-22T17:15:36,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/4056fe9be87e4fc58effa66438610fbe is 50, key is test_row_0/C:col10/1732295736206/Put/seqid=0 2024-11-22T17:15:36,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742056_1232 (size=12151) 2024-11-22T17:15:36,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/4056fe9be87e4fc58effa66438610fbe 2024-11-22T17:15:36,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/e8fb01058efb47268ae389ad0904b5c7 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e8fb01058efb47268ae389ad0904b5c7 2024-11-22T17:15:36,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e8fb01058efb47268ae389ad0904b5c7, entries=200, sequenceid=185, filesize=14.2 K 2024-11-22T17:15:36,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/3d53b9a0b48648288af7861586844a42 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/3d53b9a0b48648288af7861586844a42 2024-11-22T17:15:36,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/3d53b9a0b48648288af7861586844a42, entries=150, sequenceid=185, filesize=11.9 K 2024-11-22T17:15:36,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/4056fe9be87e4fc58effa66438610fbe as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/4056fe9be87e4fc58effa66438610fbe 2024-11-22T17:15:36,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/4056fe9be87e4fc58effa66438610fbe, entries=150, sequenceid=185, filesize=11.9 K 2024-11-22T17:15:36,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 0da185830e822d28cb5b7cd0a28bc9f1 in 97ms, sequenceid=185, compaction requested=true 2024-11-22T17:15:36,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:36,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:15:36,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:36,304 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-22T17:15:36,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:15:36,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:36,304 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-22T17:15:36,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:15:36,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:36,307 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 91770 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-22T17:15:36,307 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/B is initiating minor compaction (all files) 2024-11-22T17:15:36,307 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/B in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,307 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 98840 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-22T17:15:36,307 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/A is initiating minor compaction (all files) 2024-11-22T17:15:36,307 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/a18ec83590ac432091ddbbee9dcda0f8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/7d6581e9b2484a2b825f9dc7fa7f52be, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/88caff994b274746979d3ff6993773ab, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/19bb3547648f4e2bbf8673126c77b1eb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/9e158601273e414ca1cf124e838539fa, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/83f7f61741914d3ab9889957963951af, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/3b0e2b3deaf741e1b9800c2ef106d275, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/3d53b9a0b48648288af7861586844a42] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=89.6 K 2024-11-22T17:15:36,307 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/A in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,307 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bfd21899f715453ea1216b0a7959bd06, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/ad9e179afbd6496687b0ea24d2d5114c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f98b419bb5f043529178391af022082c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/de70e72c632d4918acd2986cc98b739b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/99e2ecaa39d14e6c896ca8c01fae9f84, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/45d0ab5c96ed4571847b0005c20af773, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e3650967e2c1418a99dc2134684215ca, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e8fb01058efb47268ae389ad0904b5c7] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=96.5 K 2024-11-22T17:15:36,307 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a18ec83590ac432091ddbbee9dcda0f8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732295728866 2024-11-22T17:15:36,307 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfd21899f715453ea1216b0a7959bd06, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732295728864 2024-11-22T17:15:36,308 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d6581e9b2484a2b825f9dc7fa7f52be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732295728914 2024-11-22T17:15:36,308 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad9e179afbd6496687b0ea24d2d5114c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732295728914 2024-11-22T17:15:36,308 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 88caff994b274746979d3ff6993773ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732295730069 2024-11-22T17:15:36,308 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting f98b419bb5f043529178391af022082c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732295730069 2024-11-22T17:15:36,308 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 19bb3547648f4e2bbf8673126c77b1eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732295731278 2024-11-22T17:15:36,308 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting de70e72c632d4918acd2986cc98b739b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732295731278 2024-11-22T17:15:36,308 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e158601273e414ca1cf124e838539fa, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732295732436 2024-11-22T17:15:36,308 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99e2ecaa39d14e6c896ca8c01fae9f84, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732295732436 2024-11-22T17:15:36,309 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45d0ab5c96ed4571847b0005c20af773, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732295733630 2024-11-22T17:15:36,309 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 83f7f61741914d3ab9889957963951af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732295733630 2024-11-22T17:15:36,310 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b0e2b3deaf741e1b9800c2ef106d275, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732295734757 2024-11-22T17:15:36,310 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3650967e2c1418a99dc2134684215ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732295734757 2024-11-22T17:15:36,310 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d53b9a0b48648288af7861586844a42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732295735885 2024-11-22T17:15:36,310 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8fb01058efb47268ae389ad0904b5c7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732295735867 2024-11-22T17:15:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T17:15:36,323 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#A#compaction#192 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:36,323 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/f8b6ce8a731840fbb43ca542432f4d22 is 50, key is test_row_0/A:col10/1732295736206/Put/seqid=0 2024-11-22T17:15:36,324 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#B#compaction#193 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:36,325 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/60ddc34ac8fa43d6aae66d4f9cedb246 is 50, key is test_row_0/B:col10/1732295736206/Put/seqid=0 2024-11-22T17:15:36,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742057_1233 (size=12425) 2024-11-22T17:15:36,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742058_1234 (size=12425) 2024-11-22T17:15:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:36,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T17:15:36,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:36,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:36,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:36,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:36,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:36,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:36,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T17:15:36,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:36,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/cb57942fc1c74274aa58563f16b4ebeb is 50, key is test_row_0/A:col10/1732295736224/Put/seqid=0 2024-11-22T17:15:36,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742059_1235 (size=12147) 2024-11-22T17:15:36,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295796389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295796391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295736384, max timestamp=1732295736394Num uniques 859; 2024-11-22T17:15:36,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295736384, max timestamp=1732295736394Num uniques 859; 2024-11-22T17:15:36,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295736384, max timestamp=1732295736394Num uniques 859; 2024-11-22T17:15:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:36,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:36,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295796497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295796501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295796501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T17:15:36,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:36,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T17:15:36,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295796611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T17:15:36,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:36,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295796724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295796724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,747 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/f8b6ce8a731840fbb43ca542432f4d22 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f8b6ce8a731840fbb43ca542432f4d22 2024-11-22T17:15:36,759 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/60ddc34ac8fa43d6aae66d4f9cedb246 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/60ddc34ac8fa43d6aae66d4f9cedb246 2024-11-22T17:15:36,765 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/A of 0da185830e822d28cb5b7cd0a28bc9f1 into f8b6ce8a731840fbb43ca542432f4d22(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:36,765 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:36,765 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/A, priority=8, startTime=1732295736304; duration=0sec 2024-11-22T17:15:36,765 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:36,765 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:A 2024-11-22T17:15:36,765 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-22T17:15:36,767 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/B of 0da185830e822d28cb5b7cd0a28bc9f1 into 60ddc34ac8fa43d6aae66d4f9cedb246(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:36,767 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:36,767 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/B, priority=8, startTime=1732295736304; duration=0sec 2024-11-22T17:15:36,768 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:36,768 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:B 2024-11-22T17:15:36,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/cb57942fc1c74274aa58563f16b4ebeb 2024-11-22T17:15:36,771 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 91770 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-22T17:15:36,771 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/C is initiating minor compaction (all files) 2024-11-22T17:15:36,771 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/C in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,771 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/be28215cbd0449c2898c3268c1fed432, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/2739783baa624217954e61293410c8b9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/754d826347e94adaac13bb6549823acb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/46d9e5c5304b4a6ebc34a97732eff88a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/068cba49975d4ffbb94ec36f4f95151a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/e96ee8c497ee40c2936b5990a5e377ef, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/5ca3a8c4520344398510a8ef11b8e54e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/4056fe9be87e4fc58effa66438610fbe] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=89.6 K 2024-11-22T17:15:36,772 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting be28215cbd0449c2898c3268c1fed432, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732295728866 2024-11-22T17:15:36,772 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2739783baa624217954e61293410c8b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732295728914 2024-11-22T17:15:36,773 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 754d826347e94adaac13bb6549823acb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732295730069 2024-11-22T17:15:36,773 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46d9e5c5304b4a6ebc34a97732eff88a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732295731278 2024-11-22T17:15:36,773 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 068cba49975d4ffbb94ec36f4f95151a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732295732436 2024-11-22T17:15:36,774 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting e96ee8c497ee40c2936b5990a5e377ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732295733630 2024-11-22T17:15:36,774 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ca3a8c4520344398510a8ef11b8e54e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732295734757 2024-11-22T17:15:36,774 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4056fe9be87e4fc58effa66438610fbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732295735885 2024-11-22T17:15:36,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/28eb6b2bb7cc4789b4aba09ad236864c is 50, key is test_row_0/B:col10/1732295736224/Put/seqid=0 2024-11-22T17:15:36,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742060_1236 (size=9757) 2024-11-22T17:15:36,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/28eb6b2bb7cc4789b4aba09ad236864c 2024-11-22T17:15:36,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/0d135a8e433e4244821c19bd62ecb55a is 50, key is test_row_0/C:col10/1732295736224/Put/seqid=0 2024-11-22T17:15:36,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742061_1237 (size=9757) 2024-11-22T17:15:36,821 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#C#compaction#197 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:36,821 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/603f6c2cbba745858dc262b69eeeda92 is 50, key is test_row_0/C:col10/1732295736206/Put/seqid=0 2024-11-22T17:15:36,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T17:15:36,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295796822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:36,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/0d135a8e433e4244821c19bd62ecb55a 2024-11-22T17:15:36,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/cb57942fc1c74274aa58563f16b4ebeb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/cb57942fc1c74274aa58563f16b4ebeb 2024-11-22T17:15:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T17:15:36,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/cb57942fc1c74274aa58563f16b4ebeb, entries=150, sequenceid=196, filesize=11.9 K 2024-11-22T17:15:36,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/28eb6b2bb7cc4789b4aba09ad236864c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/28eb6b2bb7cc4789b4aba09ad236864c 2024-11-22T17:15:36,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/28eb6b2bb7cc4789b4aba09ad236864c, entries=100, sequenceid=196, filesize=9.5 K 2024-11-22T17:15:36,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/0d135a8e433e4244821c19bd62ecb55a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/0d135a8e433e4244821c19bd62ecb55a 2024-11-22T17:15:36,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742062_1238 (size=12425) 2024-11-22T17:15:36,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/0d135a8e433e4244821c19bd62ecb55a, entries=100, sequenceid=196, filesize=9.5 K 2024-11-22T17:15:36,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0da185830e822d28cb5b7cd0a28bc9f1 in 503ms, sequenceid=196, compaction requested=false 2024-11-22T17:15:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:36,853 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/603f6c2cbba745858dc262b69eeeda92 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/603f6c2cbba745858dc262b69eeeda92 2024-11-22T17:15:36,858 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/C of 0da185830e822d28cb5b7cd0a28bc9f1 into 603f6c2cbba745858dc262b69eeeda92(size=12.1 K), total size for store is 21.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:36,858 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:36,858 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/C, priority=8, startTime=1732295736304; duration=0sec 2024-11-22T17:15:36,858 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:36,858 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:C 2024-11-22T17:15:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:36,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-22T17:15:36,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:36,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:36,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:36,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:36,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:36,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:36,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/76c0eead5a0a4f51be14dfb0004984eb is 50, key is test_row_0/A:col10/1732295736453/Put/seqid=0 2024-11-22T17:15:36,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T17:15:36,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:36,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:36,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:36,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295796980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:36,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295796984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:36,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742063_1239 (size=14541) 2024-11-22T17:15:36,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/76c0eead5a0a4f51be14dfb0004984eb 2024-11-22T17:15:37,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/708fd9206db947e984ea4732ac8e9f32 is 50, key is test_row_0/B:col10/1732295736453/Put/seqid=0 2024-11-22T17:15:37,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295797032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:37,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295797041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:37,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742064_1240 (size=12151) 2024-11-22T17:15:37,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295797102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295797102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:37,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T17:15:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295797132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:37,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T17:15:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295797311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:37,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295797312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T17:15:37,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T17:15:37,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:37,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/708fd9206db947e984ea4732ac8e9f32 2024-11-22T17:15:37,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/3bf007b356184a6fa0d0ca026d9b32b6 is 50, key is test_row_0/C:col10/1732295736453/Put/seqid=0 2024-11-22T17:15:37,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742065_1241 (size=12151) 2024-11-22T17:15:37,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295797541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:37,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295797552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:37,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295797622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:37,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295797623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:37,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295797642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:37,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T17:15:37,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:37,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T17:15:37,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:37,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/3bf007b356184a6fa0d0ca026d9b32b6 2024-11-22T17:15:37,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/76c0eead5a0a4f51be14dfb0004984eb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/76c0eead5a0a4f51be14dfb0004984eb 2024-11-22T17:15:37,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/76c0eead5a0a4f51be14dfb0004984eb, entries=200, sequenceid=235, filesize=14.2 K 2024-11-22T17:15:37,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/708fd9206db947e984ea4732ac8e9f32 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/708fd9206db947e984ea4732ac8e9f32 2024-11-22T17:15:37,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/708fd9206db947e984ea4732ac8e9f32, entries=150, sequenceid=235, filesize=11.9 K 2024-11-22T17:15:37,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/3bf007b356184a6fa0d0ca026d9b32b6 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3bf007b356184a6fa0d0ca026d9b32b6 2024-11-22T17:15:37,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T17:15:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:37,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3bf007b356184a6fa0d0ca026d9b32b6, entries=150, sequenceid=235, filesize=11.9 K 2024-11-22T17:15:37,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 0da185830e822d28cb5b7cd0a28bc9f1 in 1041ms, sequenceid=235, compaction requested=true 2024-11-22T17:15:37,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:37,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:15:37,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:37,989 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:37,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:15:37,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:37,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:15:37,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T17:15:37,989 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:37,991 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:37,991 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/B is initiating minor compaction (all files) 2024-11-22T17:15:37,991 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/B in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,991 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/60ddc34ac8fa43d6aae66d4f9cedb246, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/28eb6b2bb7cc4789b4aba09ad236864c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/708fd9206db947e984ea4732ac8e9f32] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=33.5 K 2024-11-22T17:15:37,991 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39113 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:37,991 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/A is initiating minor compaction (all files) 2024-11-22T17:15:37,992 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/A in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:37,992 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f8b6ce8a731840fbb43ca542432f4d22, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/cb57942fc1c74274aa58563f16b4ebeb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/76c0eead5a0a4f51be14dfb0004984eb] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=38.2 K 2024-11-22T17:15:37,993 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ddc34ac8fa43d6aae66d4f9cedb246, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732295735885 2024-11-22T17:15:37,993 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8b6ce8a731840fbb43ca542432f4d22, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732295735885 2024-11-22T17:15:37,993 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 28eb6b2bb7cc4789b4aba09ad236864c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732295736224 2024-11-22T17:15:37,996 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb57942fc1c74274aa58563f16b4ebeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732295736224 2024-11-22T17:15:37,997 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 708fd9206db947e984ea4732ac8e9f32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732295736453 2024-11-22T17:15:37,997 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76c0eead5a0a4f51be14dfb0004984eb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732295736453 2024-11-22T17:15:38,022 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#B#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:38,023 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/d6642a94dc0c4425a7bd769c881db16c is 50, key is test_row_0/B:col10/1732295736453/Put/seqid=0 2024-11-22T17:15:38,033 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#A#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:38,034 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/bbb68560d49441398c9b19416e200588 is 50, key is test_row_0/A:col10/1732295736453/Put/seqid=0 2024-11-22T17:15:38,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742066_1242 (size=12527) 2024-11-22T17:15:38,087 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/d6642a94dc0c4425a7bd769c881db16c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/d6642a94dc0c4425a7bd769c881db16c 2024-11-22T17:15:38,097 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/B of 0da185830e822d28cb5b7cd0a28bc9f1 into d6642a94dc0c4425a7bd769c881db16c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:38,097 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:38,097 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/B, priority=13, startTime=1732295737989; duration=0sec 2024-11-22T17:15:38,097 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:38,097 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:B 2024-11-22T17:15:38,097 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:38,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742067_1243 (size=12527) 2024-11-22T17:15:38,100 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:38,100 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/C is initiating minor compaction (all files) 2024-11-22T17:15:38,101 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/C in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:38,101 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/603f6c2cbba745858dc262b69eeeda92, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/0d135a8e433e4244821c19bd62ecb55a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3bf007b356184a6fa0d0ca026d9b32b6] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=33.5 K 2024-11-22T17:15:38,105 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 603f6c2cbba745858dc262b69eeeda92, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732295735885 2024-11-22T17:15:38,108 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d135a8e433e4244821c19bd62ecb55a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732295736224 2024-11-22T17:15:38,111 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/bbb68560d49441398c9b19416e200588 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bbb68560d49441398c9b19416e200588 2024-11-22T17:15:38,116 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bf007b356184a6fa0d0ca026d9b32b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732295736453 2024-11-22T17:15:38,120 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/A of 0da185830e822d28cb5b7cd0a28bc9f1 into bbb68560d49441398c9b19416e200588(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:38,120 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:38,120 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/A, priority=13, startTime=1732295737989; duration=0sec 2024-11-22T17:15:38,120 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:38,120 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:A 2024-11-22T17:15:38,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T17:15:38,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:38,136 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T17:15:38,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:38,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:38,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:38,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:38,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:38,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:38,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:38,143 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#C#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:38,144 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/50c6aa246c174ba7aeba59e9c4fb04ae is 50, key is test_row_0/C:col10/1732295736453/Put/seqid=0 2024-11-22T17:15:38,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/e82176bb410742c4b57339fc24bcc3ca is 50, key is test_row_0/A:col10/1732295736970/Put/seqid=0 2024-11-22T17:15:38,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742068_1244 (size=12527) 2024-11-22T17:15:38,241 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/50c6aa246c174ba7aeba59e9c4fb04ae as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/50c6aa246c174ba7aeba59e9c4fb04ae 2024-11-22T17:15:38,254 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/C of 0da185830e822d28cb5b7cd0a28bc9f1 into 50c6aa246c174ba7aeba59e9c4fb04ae(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:38,254 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:38,254 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/C, priority=13, startTime=1732295737989; duration=0sec 2024-11-22T17:15:38,255 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:38,257 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:C 2024-11-22T17:15:38,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742069_1245 (size=12151) 2024-11-22T17:15:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295798273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:38,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295798275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T17:15:38,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295798381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:38,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295798391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:38,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295798551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:38,553 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 0da185830e822d28cb5b7cd0a28bc9f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:38,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295798571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:38,574 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_0' in 0da185830e822d28cb5b7cd0a28bc9f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:38,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295798592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:38,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295798601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:38,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295798651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:38,654 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 0da185830e822d28cb5b7cd0a28bc9f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:38,660 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/e82176bb410742c4b57339fc24bcc3ca 2024-11-22T17:15:38,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/14601dec565346c29f4f28aa825fcc35 is 50, key is test_row_0/B:col10/1732295736970/Put/seqid=0 2024-11-22T17:15:38,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742070_1246 (size=12151) 2024-11-22T17:15:38,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295798901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:38,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295798912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:39,075 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/14601dec565346c29f4f28aa825fcc35 2024-11-22T17:15:39,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/41af56bc711a45288e70677017361203 is 50, key is test_row_0/C:col10/1732295736970/Put/seqid=0 2024-11-22T17:15:39,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742071_1247 (size=12151) 2024-11-22T17:15:39,127 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/41af56bc711a45288e70677017361203 2024-11-22T17:15:39,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/e82176bb410742c4b57339fc24bcc3ca as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e82176bb410742c4b57339fc24bcc3ca 2024-11-22T17:15:39,146 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e82176bb410742c4b57339fc24bcc3ca, entries=150, sequenceid=249, filesize=11.9 K 2024-11-22T17:15:39,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/14601dec565346c29f4f28aa825fcc35 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/14601dec565346c29f4f28aa825fcc35 2024-11-22T17:15:39,152 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/14601dec565346c29f4f28aa825fcc35, entries=150, sequenceid=249, filesize=11.9 K 2024-11-22T17:15:39,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/41af56bc711a45288e70677017361203 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/41af56bc711a45288e70677017361203 2024-11-22T17:15:39,156 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/41af56bc711a45288e70677017361203, entries=150, sequenceid=249, filesize=11.9 K 2024-11-22T17:15:39,157 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0da185830e822d28cb5b7cd0a28bc9f1 in 1021ms, sequenceid=249, compaction requested=false 2024-11-22T17:15:39,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:39,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:39,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-22T17:15:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-22T17:15:39,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-22T17:15:39,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9530 sec 2024-11-22T17:15:39,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.9580 sec 2024-11-22T17:15:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:39,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295738247, max timestamp=1732295739413Num uniques 859; 2024-11-22T17:15:39,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:15:39,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-22T17:15:39,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:39,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:39,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:39,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:39,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:39,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:39,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/d7ea55de53bc40509c92bd9691fde004 is 50, key is test_row_0/A:col10/1732295739413/Put/seqid=0 2024-11-22T17:15:39,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742072_1248 (size=12301) 2024-11-22T17:15:39,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295799456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:39,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295799458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:39,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295799572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:39,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295799572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:39,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295799781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:39,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295799781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:39,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/d7ea55de53bc40509c92bd9691fde004 2024-11-22T17:15:39,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/377292ccb91e4240acd2af4affbe3bcb is 50, key is test_row_0/B:col10/1732295739413/Put/seqid=0 2024-11-22T17:15:39,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742073_1249 (size=12301) 2024-11-22T17:15:40,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295800091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295800091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/377292ccb91e4240acd2af4affbe3bcb 2024-11-22T17:15:40,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/32d3dee3484b432c95aa9cce23db207c is 50, key is test_row_0/C:col10/1732295739413/Put/seqid=0 2024-11-22T17:15:40,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742074_1250 (size=12301) 2024-11-22T17:15:40,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/32d3dee3484b432c95aa9cce23db207c 2024-11-22T17:15:40,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/d7ea55de53bc40509c92bd9691fde004 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/d7ea55de53bc40509c92bd9691fde004 2024-11-22T17:15:40,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/d7ea55de53bc40509c92bd9691fde004, entries=150, sequenceid=271, filesize=12.0 K 2024-11-22T17:15:40,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/377292ccb91e4240acd2af4affbe3bcb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/377292ccb91e4240acd2af4affbe3bcb 2024-11-22T17:15:40,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/377292ccb91e4240acd2af4affbe3bcb, entries=150, sequenceid=271, filesize=12.0 K 2024-11-22T17:15:40,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/32d3dee3484b432c95aa9cce23db207c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/32d3dee3484b432c95aa9cce23db207c 2024-11-22T17:15:40,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/32d3dee3484b432c95aa9cce23db207c, entries=150, sequenceid=271, filesize=12.0 K 2024-11-22T17:15:40,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 0da185830e822d28cb5b7cd0a28bc9f1 in 937ms, sequenceid=271, compaction requested=true 2024-11-22T17:15:40,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:40,354 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:40,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:15:40,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:40,355 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:40,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:15:40,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:40,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:15:40,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:40,358 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:40,358 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:40,358 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/B is initiating minor compaction (all files) 2024-11-22T17:15:40,358 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/A is initiating minor compaction (all files) 2024-11-22T17:15:40,358 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/B in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:40,358 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/A in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:40,358 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bbb68560d49441398c9b19416e200588, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e82176bb410742c4b57339fc24bcc3ca, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/d7ea55de53bc40509c92bd9691fde004] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=36.1 K 2024-11-22T17:15:40,358 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/d6642a94dc0c4425a7bd769c881db16c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/14601dec565346c29f4f28aa825fcc35, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/377292ccb91e4240acd2af4affbe3bcb] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=36.1 K 2024-11-22T17:15:40,358 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting d6642a94dc0c4425a7bd769c881db16c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732295736453 2024-11-22T17:15:40,358 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbb68560d49441398c9b19416e200588, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732295736453 2024-11-22T17:15:40,359 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 14601dec565346c29f4f28aa825fcc35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732295736953 2024-11-22T17:15:40,359 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 377292ccb91e4240acd2af4affbe3bcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732295738247 2024-11-22T17:15:40,360 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting e82176bb410742c4b57339fc24bcc3ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732295736953 2024-11-22T17:15:40,360 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7ea55de53bc40509c92bd9691fde004, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732295738247 2024-11-22T17:15:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T17:15:40,362 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:40,362 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-22T17:15:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T17:15:40,366 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:40,366 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:40,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:40,375 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#B#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:40,375 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/d2ffeaefa07c4b2c965a97df1e6812fd is 50, key is test_row_0/B:col10/1732295739413/Put/seqid=0 2024-11-22T17:15:40,378 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#A#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:40,379 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/cce6608c22eb4bbfa27c02c2a6e2a9c7 is 50, key is test_row_0/A:col10/1732295739413/Put/seqid=0 2024-11-22T17:15:40,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742076_1252 (size=12779) 2024-11-22T17:15:40,407 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/cce6608c22eb4bbfa27c02c2a6e2a9c7 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/cce6608c22eb4bbfa27c02c2a6e2a9c7 2024-11-22T17:15:40,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742075_1251 (size=12779) 2024-11-22T17:15:40,421 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/A of 0da185830e822d28cb5b7cd0a28bc9f1 into cce6608c22eb4bbfa27c02c2a6e2a9c7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:40,421 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:40,421 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/A, priority=13, startTime=1732295740354; duration=0sec 2024-11-22T17:15:40,421 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:40,421 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:A 2024-11-22T17:15:40,421 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:40,423 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:40,423 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/C is initiating minor compaction (all files) 2024-11-22T17:15:40,423 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/C in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:40,423 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/50c6aa246c174ba7aeba59e9c4fb04ae, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/41af56bc711a45288e70677017361203, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/32d3dee3484b432c95aa9cce23db207c] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=36.1 K 2024-11-22T17:15:40,423 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50c6aa246c174ba7aeba59e9c4fb04ae, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732295736453 2024-11-22T17:15:40,424 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41af56bc711a45288e70677017361203, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732295736953 2024-11-22T17:15:40,424 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32d3dee3484b432c95aa9cce23db207c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732295738247 2024-11-22T17:15:40,425 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/d2ffeaefa07c4b2c965a97df1e6812fd as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/d2ffeaefa07c4b2c965a97df1e6812fd 2024-11-22T17:15:40,432 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/B of 0da185830e822d28cb5b7cd0a28bc9f1 into d2ffeaefa07c4b2c965a97df1e6812fd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:40,432 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:40,432 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/B, priority=13, startTime=1732295740355; duration=0sec 2024-11-22T17:15:40,432 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:40,432 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:B 2024-11-22T17:15:40,441 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:40,442 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/57dbf55b029d458aa31ac69eca035cd2 is 50, key is test_row_0/C:col10/1732295739413/Put/seqid=0 2024-11-22T17:15:40,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742077_1253 (size=12779) 2024-11-22T17:15:40,465 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/57dbf55b029d458aa31ac69eca035cd2 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/57dbf55b029d458aa31ac69eca035cd2 2024-11-22T17:15:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T17:15:40,474 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/C of 0da185830e822d28cb5b7cd0a28bc9f1 into 57dbf55b029d458aa31ac69eca035cd2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:40,474 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:40,474 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/C, priority=13, startTime=1732295740355; duration=0sec 2024-11-22T17:15:40,474 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:40,474 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:C 2024-11-22T17:15:40,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-22T17:15:40,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:40,520 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-22T17:15:40,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:40,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:40,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:40,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:40,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:40,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/f59f2d0df367428982c71e5e826ffe8b is 50, key is test_row_0/A:col10/1732295739456/Put/seqid=0 2024-11-22T17:15:40,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742078_1254 (size=12301) 2024-11-22T17:15:40,533 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/f59f2d0df367428982c71e5e826ffe8b 2024-11-22T17:15:40,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/5d365cc6e9984711b5e04f31496c5691 is 50, key is test_row_0/B:col10/1732295739456/Put/seqid=0 2024-11-22T17:15:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:40,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:40,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742079_1255 (size=12301) 2024-11-22T17:15:40,588 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/5d365cc6e9984711b5e04f31496c5691 2024-11-22T17:15:40,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/79950141c14340afb6b0f6e4dad18098 is 50, key is test_row_0/C:col10/1732295739456/Put/seqid=0 2024-11-22T17:15:40,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742080_1256 (size=12301) 2024-11-22T17:15:40,611 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/79950141c14340afb6b0f6e4dad18098 2024-11-22T17:15:40,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/f59f2d0df367428982c71e5e826ffe8b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f59f2d0df367428982c71e5e826ffe8b 2024-11-22T17:15:40,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295800611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295800617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,624 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f59f2d0df367428982c71e5e826ffe8b, entries=150, sequenceid=290, filesize=12.0 K 2024-11-22T17:15:40,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295800619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295800621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/5d365cc6e9984711b5e04f31496c5691 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/5d365cc6e9984711b5e04f31496c5691 2024-11-22T17:15:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,630 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/5d365cc6e9984711b5e04f31496c5691, entries=150, sequenceid=290, filesize=12.0 K 2024-11-22T17:15:40,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/79950141c14340afb6b0f6e4dad18098 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/79950141c14340afb6b0f6e4dad18098 2024-11-22T17:15:40,634 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/79950141c14340afb6b0f6e4dad18098, entries=150, sequenceid=290, filesize=12.0 K 2024-11-22T17:15:40,635 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 0da185830e822d28cb5b7cd0a28bc9f1 in 115ms, sequenceid=290, compaction requested=false 2024-11-22T17:15:40,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:40,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:40,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-22T17:15:40,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-22T17:15:40,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-22T17:15:40,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 270 msec 2024-11-22T17:15:40,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 279 msec 2024-11-22T17:15:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T17:15:40,681 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:40,682 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-22T17:15:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T17:15:40,683 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:40,684 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:40,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:40,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T17:15:40,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:40,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:40,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:40,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:40,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:40,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:40,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/dc0ab77d2034489692a5b0ba39c6dba4 is 50, key is test_row_0/A:col10/1732295740618/Put/seqid=0 2024-11-22T17:15:40,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742081_1257 (size=14741) 2024-11-22T17:15:40,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295800732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295800735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295800736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295800737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295800737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T17:15:40,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T17:15:40,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:40,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:40,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:40,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:40,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:40,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295800851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295800851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295800851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295800851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295800941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:40,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T17:15:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:40,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T17:15:41,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295801061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295801062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295801062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295801062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/dc0ab77d2034489692a5b0ba39c6dba4 2024-11-22T17:15:41,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/4d251e632e9b4fde809c787fbcd7d1ae is 50, key is test_row_0/B:col10/1732295740618/Put/seqid=0 2024-11-22T17:15:41,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742082_1258 (size=12301) 2024-11-22T17:15:41,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T17:15:41,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:41,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:41,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:41,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:41,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:41,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295801251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T17:15:41,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:41,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:41,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:41,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:41,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T17:15:41,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295801371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295801372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295801372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295801372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T17:15:41,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:41,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:41,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:41,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:41,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:41,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/4d251e632e9b4fde809c787fbcd7d1ae 2024-11-22T17:15:41,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/72fcd3c4771340938e176261203af5a0 is 50, key is test_row_0/C:col10/1732295740618/Put/seqid=0 2024-11-22T17:15:41,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742083_1259 (size=12301) 2024-11-22T17:15:41,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/72fcd3c4771340938e176261203af5a0 2024-11-22T17:15:41,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/dc0ab77d2034489692a5b0ba39c6dba4 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/dc0ab77d2034489692a5b0ba39c6dba4 2024-11-22T17:15:41,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/dc0ab77d2034489692a5b0ba39c6dba4, entries=200, sequenceid=308, filesize=14.4 K 2024-11-22T17:15:41,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/4d251e632e9b4fde809c787fbcd7d1ae as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/4d251e632e9b4fde809c787fbcd7d1ae 2024-11-22T17:15:41,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/4d251e632e9b4fde809c787fbcd7d1ae, entries=150, sequenceid=308, filesize=12.0 K 2024-11-22T17:15:41,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/72fcd3c4771340938e176261203af5a0 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/72fcd3c4771340938e176261203af5a0 2024-11-22T17:15:41,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/72fcd3c4771340938e176261203af5a0, entries=150, sequenceid=308, filesize=12.0 K 2024-11-22T17:15:41,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 0da185830e822d28cb5b7cd0a28bc9f1 in 884ms, sequenceid=308, compaction requested=true 2024-11-22T17:15:41,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:41,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:15:41,571 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:41,571 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:15:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:15:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:41,573 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:41,573 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/B is initiating minor compaction (all files) 2024-11-22T17:15:41,573 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/B in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:41,574 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/d2ffeaefa07c4b2c965a97df1e6812fd, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/5d365cc6e9984711b5e04f31496c5691, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/4d251e632e9b4fde809c787fbcd7d1ae] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=36.5 K 2024-11-22T17:15:41,574 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:41,574 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/A is initiating minor compaction (all files) 2024-11-22T17:15:41,574 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/A in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:41,574 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/cce6608c22eb4bbfa27c02c2a6e2a9c7, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f59f2d0df367428982c71e5e826ffe8b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/dc0ab77d2034489692a5b0ba39c6dba4] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=38.9 K 2024-11-22T17:15:41,574 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting d2ffeaefa07c4b2c965a97df1e6812fd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732295738247 2024-11-22T17:15:41,574 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting cce6608c22eb4bbfa27c02c2a6e2a9c7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732295738247 2024-11-22T17:15:41,581 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d365cc6e9984711b5e04f31496c5691, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732295739446 2024-11-22T17:15:41,581 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting f59f2d0df367428982c71e5e826ffe8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732295739446 2024-11-22T17:15:41,581 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc0ab77d2034489692a5b0ba39c6dba4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732295740614 2024-11-22T17:15:41,582 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d251e632e9b4fde809c787fbcd7d1ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732295740617 2024-11-22T17:15:41,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T17:15:41,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:41,604 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-22T17:15:41,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:41,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:41,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:41,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:41,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:41,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:41,610 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:41,610 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:41,610 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/af920fb7085f401a91f0a790a0ef94bb is 50, key is test_row_0/B:col10/1732295740618/Put/seqid=0 2024-11-22T17:15:41,610 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/bc7b955de6684b5a8ee5506ae2e4b596 is 50, key is test_row_0/A:col10/1732295740618/Put/seqid=0 2024-11-22T17:15:41,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/6a7bf0639e044ac7b0b0c135d9947258 is 50, key is test_row_0/A:col10/1732295740728/Put/seqid=0 2024-11-22T17:15:41,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742084_1260 (size=12881) 2024-11-22T17:15:41,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742085_1261 (size=12881) 2024-11-22T17:15:41,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742086_1262 (size=12301) 2024-11-22T17:15:41,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T17:15:41,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295801860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295801882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295801882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295801882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295801882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:41,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295801971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:42,076 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/bc7b955de6684b5a8ee5506ae2e4b596 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bc7b955de6684b5a8ee5506ae2e4b596 2024-11-22T17:15:42,076 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/af920fb7085f401a91f0a790a0ef94bb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/af920fb7085f401a91f0a790a0ef94bb 2024-11-22T17:15:42,089 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/B of 0da185830e822d28cb5b7cd0a28bc9f1 into af920fb7085f401a91f0a790a0ef94bb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:42,089 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:42,089 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/B, priority=13, startTime=1732295741571; duration=0sec 2024-11-22T17:15:42,089 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/A of 0da185830e822d28cb5b7cd0a28bc9f1 into bc7b955de6684b5a8ee5506ae2e4b596(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:42,089 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:42,089 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/A, priority=13, startTime=1732295741570; duration=0sec 2024-11-22T17:15:42,089 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:42,089 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:B 2024-11-22T17:15:42,089 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:42,090 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:A 2024-11-22T17:15:42,090 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:42,095 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/6a7bf0639e044ac7b0b0c135d9947258 2024-11-22T17:15:42,095 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:42,095 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/C is initiating minor compaction (all files) 2024-11-22T17:15:42,096 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/C in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:42,096 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/57dbf55b029d458aa31ac69eca035cd2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/79950141c14340afb6b0f6e4dad18098, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/72fcd3c4771340938e176261203af5a0] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=36.5 K 2024-11-22T17:15:42,096 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 57dbf55b029d458aa31ac69eca035cd2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732295738247 2024-11-22T17:15:42,097 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 79950141c14340afb6b0f6e4dad18098, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732295739446 2024-11-22T17:15:42,097 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 72fcd3c4771340938e176261203af5a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732295740617 2024-11-22T17:15:42,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/12c353d4d6574dcea106ea6599ba1630 is 50, key is test_row_0/B:col10/1732295740728/Put/seqid=0 2024-11-22T17:15:42,130 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#C#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:42,131 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/3a7e6320ad9745caa1d79940a21c4184 is 50, key is test_row_0/C:col10/1732295740618/Put/seqid=0 2024-11-22T17:15:42,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742087_1263 (size=12301) 2024-11-22T17:15:42,163 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/12c353d4d6574dcea106ea6599ba1630 2024-11-22T17:15:42,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742088_1264 (size=12881) 2024-11-22T17:15:42,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/c26fef4dd1be4dab9dd9e56bfb292bf6 is 50, key is test_row_0/C:col10/1732295740728/Put/seqid=0 2024-11-22T17:15:42,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295802182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:42,188 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/3a7e6320ad9745caa1d79940a21c4184 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3a7e6320ad9745caa1d79940a21c4184 2024-11-22T17:15:42,197 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/C of 0da185830e822d28cb5b7cd0a28bc9f1 into 3a7e6320ad9745caa1d79940a21c4184(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:42,197 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:42,199 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/C, priority=13, startTime=1732295741571; duration=0sec 2024-11-22T17:15:42,199 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:42,199 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:C 2024-11-22T17:15:42,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742089_1265 (size=12301) 2024-11-22T17:15:42,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295802491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:42,626 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/c26fef4dd1be4dab9dd9e56bfb292bf6 2024-11-22T17:15:42,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/6a7bf0639e044ac7b0b0c135d9947258 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/6a7bf0639e044ac7b0b0c135d9947258 2024-11-22T17:15:42,641 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/6a7bf0639e044ac7b0b0c135d9947258, entries=150, sequenceid=324, filesize=12.0 K 2024-11-22T17:15:42,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/12c353d4d6574dcea106ea6599ba1630 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/12c353d4d6574dcea106ea6599ba1630 2024-11-22T17:15:42,649 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/12c353d4d6574dcea106ea6599ba1630, entries=150, sequenceid=324, filesize=12.0 K 2024-11-22T17:15:42,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/c26fef4dd1be4dab9dd9e56bfb292bf6 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/c26fef4dd1be4dab9dd9e56bfb292bf6 2024-11-22T17:15:42,663 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/c26fef4dd1be4dab9dd9e56bfb292bf6, entries=150, sequenceid=324, filesize=12.0 K 2024-11-22T17:15:42,666 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 0da185830e822d28cb5b7cd0a28bc9f1 in 1062ms, sequenceid=324, compaction requested=false 2024-11-22T17:15:42,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:42,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:42,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-22T17:15:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-22T17:15:42,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-22T17:15:42,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9850 sec 2024-11-22T17:15:42,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9910 sec 2024-11-22T17:15:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T17:15:42,832 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:42,832 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-22T17:15:42,834 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T17:15:42,834 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:42,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:42,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-22T17:15:42,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:42,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:42,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:42,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:42,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:42,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:42,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/85857bfdc5274d38abcd3b96e02b83c7 is 50, key is test_row_0/A:col10/1732295742894/Put/seqid=0 2024-11-22T17:15:42,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742090_1266 (size=17181) 2024-11-22T17:15:42,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:42,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295802911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:42,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295802913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:42,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295802914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:42,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295802916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T17:15:42,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T17:15:42,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:42,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:42,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:42,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:43,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295803011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:43,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295803022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:43,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295803022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:43,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:43,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:43,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295803031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:43,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295803031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:43,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T17:15:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:43,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T17:15:43,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295803231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:43,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295803242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:43,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:43,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295803242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295803242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:43,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T17:15:43,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:43,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:43,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:43,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:43,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:43,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/85857bfdc5274d38abcd3b96e02b83c7 2024-11-22T17:15:43,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/c8dc4a666de44d2fa24c5a933b33e059 is 50, key is test_row_0/B:col10/1732295742894/Put/seqid=0 2024-11-22T17:15:43,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742091_1267 (size=12301) 2024-11-22T17:15:43,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T17:15:43,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:43,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:43,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:43,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:43,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T17:15:43,515 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T17:15:43,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295803541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:43,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:43,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295803551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:43,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295803551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:43,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295803561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:43,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T17:15:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:43,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:43,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:43,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:43,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:43,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/c8dc4a666de44d2fa24c5a933b33e059 2024-11-22T17:15:43,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/e15ead38ca9944e89b0fd93e6dd34f9d is 50, key is test_row_0/C:col10/1732295742894/Put/seqid=0 2024-11-22T17:15:43,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742092_1268 (size=12301) 2024-11-22T17:15:43,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/e15ead38ca9944e89b0fd93e6dd34f9d 2024-11-22T17:15:43,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/85857bfdc5274d38abcd3b96e02b83c7 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/85857bfdc5274d38abcd3b96e02b83c7 2024-11-22T17:15:43,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/85857bfdc5274d38abcd3b96e02b83c7, entries=250, sequenceid=345, filesize=16.8 K 2024-11-22T17:15:43,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/c8dc4a666de44d2fa24c5a933b33e059 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/c8dc4a666de44d2fa24c5a933b33e059 2024-11-22T17:15:43,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/c8dc4a666de44d2fa24c5a933b33e059, entries=150, sequenceid=345, filesize=12.0 K 2024-11-22T17:15:43,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/e15ead38ca9944e89b0fd93e6dd34f9d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/e15ead38ca9944e89b0fd93e6dd34f9d 2024-11-22T17:15:43,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/e15ead38ca9944e89b0fd93e6dd34f9d, entries=150, sequenceid=345, filesize=12.0 K 2024-11-22T17:15:43,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0da185830e822d28cb5b7cd0a28bc9f1 in 856ms, sequenceid=345, compaction requested=true 2024-11-22T17:15:43,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:43,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:15:43,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:43,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:15:43,750 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:43,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:43,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:15:43,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:43,750 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:43,751 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:43,751 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:43,751 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/B is initiating minor compaction (all files) 2024-11-22T17:15:43,751 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/A is initiating minor compaction (all files) 2024-11-22T17:15:43,751 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/A in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:43,751 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/B in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:43,751 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/af920fb7085f401a91f0a790a0ef94bb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/12c353d4d6574dcea106ea6599ba1630, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/c8dc4a666de44d2fa24c5a933b33e059] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=36.6 K 2024-11-22T17:15:43,751 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bc7b955de6684b5a8ee5506ae2e4b596, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/6a7bf0639e044ac7b0b0c135d9947258, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/85857bfdc5274d38abcd3b96e02b83c7] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=41.4 K 2024-11-22T17:15:43,752 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting af920fb7085f401a91f0a790a0ef94bb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732295740617 2024-11-22T17:15:43,752 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc7b955de6684b5a8ee5506ae2e4b596, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732295740617 2024-11-22T17:15:43,752 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a7bf0639e044ac7b0b0c135d9947258, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732295740728 2024-11-22T17:15:43,752 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 12c353d4d6574dcea106ea6599ba1630, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732295740728 2024-11-22T17:15:43,753 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85857bfdc5274d38abcd3b96e02b83c7, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732295741826 2024-11-22T17:15:43,753 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting c8dc4a666de44d2fa24c5a933b33e059, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732295741849 2024-11-22T17:15:43,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T17:15:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:43,754 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T17:15:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:43,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:43,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:43,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:43,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/986d0a14d8404b4b80fc138026dccbbc is 50, key is test_row_0/A:col10/1732295742913/Put/seqid=0 2024-11-22T17:15:43,761 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:43,762 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/91cc2d0b274b40a7b65c9500fb093ac3 is 50, key is test_row_0/B:col10/1732295742894/Put/seqid=0 2024-11-22T17:15:43,764 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#A#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:43,764 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/c409d50c2bfb47f9995b58ac7a6b71c5 is 50, key is test_row_0/A:col10/1732295742894/Put/seqid=0 2024-11-22T17:15:43,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742093_1269 (size=12301) 2024-11-22T17:15:43,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742094_1270 (size=12983) 2024-11-22T17:15:43,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742095_1271 (size=12983) 2024-11-22T17:15:43,786 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/c409d50c2bfb47f9995b58ac7a6b71c5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/c409d50c2bfb47f9995b58ac7a6b71c5 2024-11-22T17:15:43,800 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/A of 0da185830e822d28cb5b7cd0a28bc9f1 into c409d50c2bfb47f9995b58ac7a6b71c5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:43,800 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:43,800 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/A, priority=13, startTime=1732295743750; duration=0sec 2024-11-22T17:15:43,801 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:43,801 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:A 2024-11-22T17:15:43,801 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:43,802 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:43,802 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/C is initiating minor compaction (all files) 2024-11-22T17:15:43,802 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/C in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:43,802 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3a7e6320ad9745caa1d79940a21c4184, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/c26fef4dd1be4dab9dd9e56bfb292bf6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/e15ead38ca9944e89b0fd93e6dd34f9d] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=36.6 K 2024-11-22T17:15:43,803 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a7e6320ad9745caa1d79940a21c4184, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732295740617 2024-11-22T17:15:43,803 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting c26fef4dd1be4dab9dd9e56bfb292bf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732295740728 2024-11-22T17:15:43,804 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting e15ead38ca9944e89b0fd93e6dd34f9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732295741849 2024-11-22T17:15:43,816 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#C#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:43,817 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/3933cb9cfeda4ab7a22e02815917a4b5 is 50, key is test_row_0/C:col10/1732295742894/Put/seqid=0 2024-11-22T17:15:43,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742096_1272 (size=12983) 2024-11-22T17:15:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T17:15:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:44,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:44,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295804065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295804066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295804068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295804071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295804072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,167 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/986d0a14d8404b4b80fc138026dccbbc 2024-11-22T17:15:44,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/8236d451e36a468da39e03ab8444c4a6 is 50, key is test_row_0/B:col10/1732295742913/Put/seqid=0 2024-11-22T17:15:44,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742097_1273 (size=12301) 2024-11-22T17:15:44,182 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/8236d451e36a468da39e03ab8444c4a6 2024-11-22T17:15:44,183 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/91cc2d0b274b40a7b65c9500fb093ac3 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/91cc2d0b274b40a7b65c9500fb093ac3 2024-11-22T17:15:44,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295804181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295804181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295804182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295804182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,189 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/B of 0da185830e822d28cb5b7cd0a28bc9f1 into 91cc2d0b274b40a7b65c9500fb093ac3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:44,189 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:44,189 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/B, priority=13, startTime=1732295743750; duration=0sec 2024-11-22T17:15:44,189 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:44,189 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:B 2024-11-22T17:15:44,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/0ca30c3c88ab4ee6a02e5f362b1fc5f6 is 50, key is test_row_0/C:col10/1732295742913/Put/seqid=0 2024-11-22T17:15:44,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742098_1274 (size=12301) 2024-11-22T17:15:44,198 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/0ca30c3c88ab4ee6a02e5f362b1fc5f6 2024-11-22T17:15:44,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/986d0a14d8404b4b80fc138026dccbbc as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/986d0a14d8404b4b80fc138026dccbbc 2024-11-22T17:15:44,209 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/986d0a14d8404b4b80fc138026dccbbc, entries=150, sequenceid=359, filesize=12.0 K 2024-11-22T17:15:44,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/8236d451e36a468da39e03ab8444c4a6 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/8236d451e36a468da39e03ab8444c4a6 2024-11-22T17:15:44,213 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/8236d451e36a468da39e03ab8444c4a6, entries=150, sequenceid=359, filesize=12.0 K 2024-11-22T17:15:44,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/0ca30c3c88ab4ee6a02e5f362b1fc5f6 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/0ca30c3c88ab4ee6a02e5f362b1fc5f6 2024-11-22T17:15:44,222 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/0ca30c3c88ab4ee6a02e5f362b1fc5f6, entries=150, sequenceid=359, filesize=12.0 K 2024-11-22T17:15:44,223 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0da185830e822d28cb5b7cd0a28bc9f1 in 469ms, sequenceid=359, compaction requested=false 2024-11-22T17:15:44,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:44,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:44,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-22T17:15:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-22T17:15:44,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-22T17:15:44,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3910 sec 2024-11-22T17:15:44,228 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/3933cb9cfeda4ab7a22e02815917a4b5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3933cb9cfeda4ab7a22e02815917a4b5 2024-11-22T17:15:44,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.3950 sec 2024-11-22T17:15:44,232 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/C of 0da185830e822d28cb5b7cd0a28bc9f1 into 3933cb9cfeda4ab7a22e02815917a4b5(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:44,233 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:44,233 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/C, priority=13, startTime=1732295743750; duration=0sec 2024-11-22T17:15:44,233 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:44,233 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:C 2024-11-22T17:15:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:44,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T17:15:44,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:44,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:44,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:44,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:44,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:44,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:44,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/565148225f15448eb79fcce6b3f3d5b8 is 50, key is test_row_0/A:col10/1732295744393/Put/seqid=0 2024-11-22T17:15:44,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742099_1275 (size=12301) 2024-11-22T17:15:44,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/565148225f15448eb79fcce6b3f3d5b8 2024-11-22T17:15:44,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295804439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295804440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295804441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295804444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/2e47c087cfa340edb97af4a4c820df46 is 50, key is test_row_0/B:col10/1732295744393/Put/seqid=0 2024-11-22T17:15:44,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742100_1276 (size=12301) 2024-11-22T17:15:44,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295804551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295804552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295804552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295804552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295804761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295804761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295804761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295804761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:44,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/2e47c087cfa340edb97af4a4c820df46 2024-11-22T17:15:44,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/20d643644d6d4c8e9abf6c6da0c902f1 is 50, key is test_row_0/C:col10/1732295744393/Put/seqid=0 2024-11-22T17:15:44,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742101_1277 (size=12301) 2024-11-22T17:15:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T17:15:44,981 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:44,982 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-22T17:15:44,984 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T17:15:44,984 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:44,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:45,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295805071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295805071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295805071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295805071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T17:15:45,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295805091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,095 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 0da185830e822d28cb5b7cd0a28bc9f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T17:15:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:45,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:45,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/20d643644d6d4c8e9abf6c6da0c902f1 2024-11-22T17:15:45,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/565148225f15448eb79fcce6b3f3d5b8 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/565148225f15448eb79fcce6b3f3d5b8 2024-11-22T17:15:45,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T17:15:45,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:45,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:45,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:45,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:45,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:45,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/565148225f15448eb79fcce6b3f3d5b8, entries=150, sequenceid=382, filesize=12.0 K 2024-11-22T17:15:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:45,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/2e47c087cfa340edb97af4a4c820df46 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/2e47c087cfa340edb97af4a4c820df46 2024-11-22T17:15:45,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/2e47c087cfa340edb97af4a4c820df46, entries=150, sequenceid=382, filesize=12.0 K 2024-11-22T17:15:45,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/20d643644d6d4c8e9abf6c6da0c902f1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/20d643644d6d4c8e9abf6c6da0c902f1 2024-11-22T17:15:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T17:15:45,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/20d643644d6d4c8e9abf6c6da0c902f1, entries=150, sequenceid=382, filesize=12.0 K 2024-11-22T17:15:45,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0da185830e822d28cb5b7cd0a28bc9f1 in 909ms, sequenceid=382, compaction requested=true 2024-11-22T17:15:45,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:45,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:15:45,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:45,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:15:45,303 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:45,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:45,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:15:45,303 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:45,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:45,304 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:45,304 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:45,304 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/A is initiating minor compaction (all files) 2024-11-22T17:15:45,304 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/B is initiating minor compaction (all files) 2024-11-22T17:15:45,304 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/B in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:45,304 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/A in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:45,304 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/91cc2d0b274b40a7b65c9500fb093ac3, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/8236d451e36a468da39e03ab8444c4a6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/2e47c087cfa340edb97af4a4c820df46] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=36.7 K 2024-11-22T17:15:45,304 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/c409d50c2bfb47f9995b58ac7a6b71c5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/986d0a14d8404b4b80fc138026dccbbc, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/565148225f15448eb79fcce6b3f3d5b8] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=36.7 K 2024-11-22T17:15:45,305 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting c409d50c2bfb47f9995b58ac7a6b71c5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732295741849 2024-11-22T17:15:45,305 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 986d0a14d8404b4b80fc138026dccbbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732295742909 2024-11-22T17:15:45,305 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 91cc2d0b274b40a7b65c9500fb093ac3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732295741849 2024-11-22T17:15:45,305 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 565148225f15448eb79fcce6b3f3d5b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732295744065 2024-11-22T17:15:45,306 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 8236d451e36a468da39e03ab8444c4a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732295742909 2024-11-22T17:15:45,306 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e47c087cfa340edb97af4a4c820df46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732295744065 2024-11-22T17:15:45,313 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:45,314 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:45,315 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/f6a567b4b3ef491a854192c6104535d2 is 50, key is test_row_0/B:col10/1732295744393/Put/seqid=0 2024-11-22T17:15:45,315 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/052aab980f6a4310a2a25607a1d1c63d is 50, key is test_row_0/A:col10/1732295744393/Put/seqid=0 2024-11-22T17:15:45,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742102_1278 (size=13085) 2024-11-22T17:15:45,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742103_1279 (size=13085) 2024-11-22T17:15:45,337 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/f6a567b4b3ef491a854192c6104535d2 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/f6a567b4b3ef491a854192c6104535d2 2024-11-22T17:15:45,343 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/B of 0da185830e822d28cb5b7cd0a28bc9f1 into f6a567b4b3ef491a854192c6104535d2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:45,343 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:45,343 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/B, priority=13, startTime=1732295745303; duration=0sec 2024-11-22T17:15:45,344 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:45,344 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:B 2024-11-22T17:15:45,344 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:45,345 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:45,345 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/C is initiating minor compaction (all files) 2024-11-22T17:15:45,345 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/C in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:45,345 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3933cb9cfeda4ab7a22e02815917a4b5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/0ca30c3c88ab4ee6a02e5f362b1fc5f6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/20d643644d6d4c8e9abf6c6da0c902f1] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=36.7 K 2024-11-22T17:15:45,346 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 3933cb9cfeda4ab7a22e02815917a4b5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732295741849 2024-11-22T17:15:45,346 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca30c3c88ab4ee6a02e5f362b1fc5f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732295742909 2024-11-22T17:15:45,346 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 20d643644d6d4c8e9abf6c6da0c902f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732295744065 2024-11-22T17:15:45,353 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:45,354 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/ac6e6941564b486fbf94c0a7952eedc8 is 50, key is test_row_0/C:col10/1732295744393/Put/seqid=0 2024-11-22T17:15:45,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742104_1280 (size=13085) 2024-11-22T17:15:45,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T17:15:45,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:45,444 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T17:15:45,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:45,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:45,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:45,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:45,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:45,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:45,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/4f757080a6a1492487ff86c055397022 is 50, key is test_row_0/A:col10/1732295744435/Put/seqid=0 2024-11-22T17:15:45,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742105_1281 (size=12301) 2024-11-22T17:15:45,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T17:15:45,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295805615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295805616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295805617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295805618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295805731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295805731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295805732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295805732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,741 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/052aab980f6a4310a2a25607a1d1c63d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/052aab980f6a4310a2a25607a1d1c63d 2024-11-22T17:15:45,752 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/A of 0da185830e822d28cb5b7cd0a28bc9f1 into 052aab980f6a4310a2a25607a1d1c63d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:45,752 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:45,752 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/A, priority=13, startTime=1732295745303; duration=0sec 2024-11-22T17:15:45,752 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:45,752 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:A 2024-11-22T17:15:45,766 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/ac6e6941564b486fbf94c0a7952eedc8 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/ac6e6941564b486fbf94c0a7952eedc8 2024-11-22T17:15:45,772 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/C of 0da185830e822d28cb5b7cd0a28bc9f1 into ac6e6941564b486fbf94c0a7952eedc8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:45,772 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:45,772 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/C, priority=13, startTime=1732295745303; duration=0sec 2024-11-22T17:15:45,772 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:45,772 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:C 2024-11-22T17:15:45,858 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/4f757080a6a1492487ff86c055397022 2024-11-22T17:15:45,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/c6078d8788eb4335b0a709eb7e42f048 is 50, key is test_row_0/B:col10/1732295744435/Put/seqid=0 2024-11-22T17:15:45,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742106_1282 (size=12301) 2024-11-22T17:15:45,873 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/c6078d8788eb4335b0a709eb7e42f048 2024-11-22T17:15:45,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/09f3413a32e54c539e4e2d0b909966b0 is 50, key is test_row_0/C:col10/1732295744435/Put/seqid=0 2024-11-22T17:15:45,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742107_1283 (size=12301) 2024-11-22T17:15:45,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295805941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295805941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295805942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295805942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T17:15:46,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295806251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295806251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295806251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295806251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:46,295 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/09f3413a32e54c539e4e2d0b909966b0 2024-11-22T17:15:46,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/4f757080a6a1492487ff86c055397022 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/4f757080a6a1492487ff86c055397022 2024-11-22T17:15:46,302 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/4f757080a6a1492487ff86c055397022, entries=150, sequenceid=395, filesize=12.0 K 2024-11-22T17:15:46,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/c6078d8788eb4335b0a709eb7e42f048 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/c6078d8788eb4335b0a709eb7e42f048 2024-11-22T17:15:46,307 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/c6078d8788eb4335b0a709eb7e42f048, entries=150, sequenceid=395, filesize=12.0 K 2024-11-22T17:15:46,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/09f3413a32e54c539e4e2d0b909966b0 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/09f3413a32e54c539e4e2d0b909966b0 2024-11-22T17:15:46,313 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/09f3413a32e54c539e4e2d0b909966b0, entries=150, sequenceid=395, filesize=12.0 K 2024-11-22T17:15:46,313 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0da185830e822d28cb5b7cd0a28bc9f1 in 869ms, sequenceid=395, compaction requested=false 2024-11-22T17:15:46,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:46,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:46,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-22T17:15:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-22T17:15:46,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-22T17:15:46,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3310 sec 2024-11-22T17:15:46,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3350 sec 2024-11-22T17:15:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:46,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-22T17:15:46,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:46,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:46,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:46,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:46,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:46,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:46,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/507b93fb26d044b1ace9968ed6f6a8e0 is 50, key is test_row_0/A:col10/1732295746763/Put/seqid=0 2024-11-22T17:15:46,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742108_1284 (size=14741) 2024-11-22T17:15:46,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295806771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:46,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295806772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:46,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295806776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:46,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:46,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295806776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:46,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:46,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295806881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295806881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295806881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:46,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295806891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:46,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:47,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295807091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:47,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:47,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295807091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295807091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:47,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295807101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:47,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:47,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:47,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1732295807121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:47,125 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 0da185830e822d28cb5b7cd0a28bc9f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:15:47,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T17:15:47,131 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:47,132 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-22T17:15:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T17:15:47,134 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:47,134 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:47,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:47,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/507b93fb26d044b1ace9968ed6f6a8e0 2024-11-22T17:15:47,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/b81a630115d84b65b9d743c0d882f97d is 50, key is test_row_0/B:col10/1732295746763/Put/seqid=0 2024-11-22T17:15:47,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742109_1285 (size=12301) 2024-11-22T17:15:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T17:15:47,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T17:15:47,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:47,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:47,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:47,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295807401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:47,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295807401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295807401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:47,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295807411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:47,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:47,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T17:15:47,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:47,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:47,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:47,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T17:15:47,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/b81a630115d84b65b9d743c0d882f97d 2024-11-22T17:15:47,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T17:15:47,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:47,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:47,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:47,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/a6373a11bb1d48539f8a0539df56b293 is 50, key is test_row_0/C:col10/1732295746763/Put/seqid=0 2024-11-22T17:15:47,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742110_1286 (size=12301) 2024-11-22T17:15:47,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T17:15:47,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:47,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:47,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:47,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T17:15:47,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T17:15:47,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:47,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. as already flushing 2024-11-22T17:15:47,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:47,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:47,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1732295807912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45794 deadline: 1732295807912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:47,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45792 deadline: 1732295807913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:47,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45786 deadline: 1732295807921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:15:48,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/a6373a11bb1d48539f8a0539df56b293 2024-11-22T17:15:48,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/507b93fb26d044b1ace9968ed6f6a8e0 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/507b93fb26d044b1ace9968ed6f6a8e0 2024-11-22T17:15:48,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/507b93fb26d044b1ace9968ed6f6a8e0, entries=200, sequenceid=419, filesize=14.4 K 2024-11-22T17:15:48,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/b81a630115d84b65b9d743c0d882f97d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/b81a630115d84b65b9d743c0d882f97d 2024-11-22T17:15:48,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/b81a630115d84b65b9d743c0d882f97d, entries=150, sequenceid=419, filesize=12.0 K 2024-11-22T17:15:48,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/a6373a11bb1d48539f8a0539df56b293 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/a6373a11bb1d48539f8a0539df56b293 2024-11-22T17:15:48,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/a6373a11bb1d48539f8a0539df56b293, entries=150, sequenceid=419, filesize=12.0 K 2024-11-22T17:15:48,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for 0da185830e822d28cb5b7cd0a28bc9f1 in 1279ms, sequenceid=419, compaction requested=true 2024-11-22T17:15:48,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:48,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:15:48,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:48,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:15:48,043 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:48,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:48,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0da185830e822d28cb5b7cd0a28bc9f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:15:48,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:48,043 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:48,043 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:48,043 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:48,044 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/A is initiating minor compaction (all files) 2024-11-22T17:15:48,044 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/B is initiating minor compaction (all files) 2024-11-22T17:15:48,044 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/A in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:48,044 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/B in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:48,044 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/f6a567b4b3ef491a854192c6104535d2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/c6078d8788eb4335b0a709eb7e42f048, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/b81a630115d84b65b9d743c0d882f97d] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=36.8 K 2024-11-22T17:15:48,044 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/052aab980f6a4310a2a25607a1d1c63d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/4f757080a6a1492487ff86c055397022, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/507b93fb26d044b1ace9968ed6f6a8e0] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=39.2 K 2024-11-22T17:15:48,044 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting f6a567b4b3ef491a854192c6104535d2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732295744065 2024-11-22T17:15:48,044 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 052aab980f6a4310a2a25607a1d1c63d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732295744065 2024-11-22T17:15:48,044 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f757080a6a1492487ff86c055397022, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732295744435 2024-11-22T17:15:48,044 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting c6078d8788eb4335b0a709eb7e42f048, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732295744435 2024-11-22T17:15:48,045 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting b81a630115d84b65b9d743c0d882f97d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732295745614 2024-11-22T17:15:48,045 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 507b93fb26d044b1ace9968ed6f6a8e0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732295745614 2024-11-22T17:15:48,051 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:48,052 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#B#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:48,052 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/eecc53ba57274e45b88c99b51c708ee5 is 50, key is test_row_0/A:col10/1732295746763/Put/seqid=0 2024-11-22T17:15:48,052 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/84e95e9420a945daaae7dca289008d17 is 50, key is test_row_0/B:col10/1732295746763/Put/seqid=0 2024-11-22T17:15:48,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T17:15:48,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:48,054 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T17:15:48,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:48,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:48,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:48,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:48,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:48,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:48,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/d5944463f8fa41e5be5b543ae92e9f2c is 50, key is test_row_0/A:col10/1732295746775/Put/seqid=0 2024-11-22T17:15:48,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742111_1287 (size=13187) 2024-11-22T17:15:48,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742113_1289 (size=12301) 2024-11-22T17:15:48,079 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/d5944463f8fa41e5be5b543ae92e9f2c 2024-11-22T17:15:48,084 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/eecc53ba57274e45b88c99b51c708ee5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/eecc53ba57274e45b88c99b51c708ee5 2024-11-22T17:15:48,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742112_1288 (size=13187) 2024-11-22T17:15:48,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/01a183e02c6b475db3d4b0253ab5b90c is 50, key is test_row_0/B:col10/1732295746775/Put/seqid=0 2024-11-22T17:15:48,092 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/A of 0da185830e822d28cb5b7cd0a28bc9f1 into eecc53ba57274e45b88c99b51c708ee5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:48,092 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:48,092 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/A, priority=13, startTime=1732295748042; duration=0sec 2024-11-22T17:15:48,092 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:48,092 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:A 2024-11-22T17:15:48,092 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:48,093 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:48,093 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 0da185830e822d28cb5b7cd0a28bc9f1/C is initiating minor compaction (all files) 2024-11-22T17:15:48,093 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0da185830e822d28cb5b7cd0a28bc9f1/C in TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:48,093 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/ac6e6941564b486fbf94c0a7952eedc8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/09f3413a32e54c539e4e2d0b909966b0, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/a6373a11bb1d48539f8a0539df56b293] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp, totalSize=36.8 K 2024-11-22T17:15:48,094 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac6e6941564b486fbf94c0a7952eedc8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732295744065 2024-11-22T17:15:48,094 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09f3413a32e54c539e4e2d0b909966b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732295744435 2024-11-22T17:15:48,095 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6373a11bb1d48539f8a0539df56b293, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732295745614 2024-11-22T17:15:48,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742114_1290 (size=12301) 2024-11-22T17:15:48,102 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0da185830e822d28cb5b7cd0a28bc9f1#C#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:48,102 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/d98d22b095fc4262abb616bccc090096 is 50, key is test_row_0/C:col10/1732295746763/Put/seqid=0 2024-11-22T17:15:48,105 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/01a183e02c6b475db3d4b0253ab5b90c 2024-11-22T17:15:48,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/158dbdee15a84e979de079e4e71f552e is 50, key is test_row_0/C:col10/1732295746775/Put/seqid=0 2024-11-22T17:15:48,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742115_1291 (size=13187) 2024-11-22T17:15:48,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742116_1292 (size=12301) 2024-11-22T17:15:48,128 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/158dbdee15a84e979de079e4e71f552e 2024-11-22T17:15:48,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/d5944463f8fa41e5be5b543ae92e9f2c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/d5944463f8fa41e5be5b543ae92e9f2c 2024-11-22T17:15:48,137 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/d5944463f8fa41e5be5b543ae92e9f2c, entries=150, sequenceid=430, filesize=12.0 K 2024-11-22T17:15:48,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/01a183e02c6b475db3d4b0253ab5b90c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/01a183e02c6b475db3d4b0253ab5b90c 2024-11-22T17:15:48,142 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/01a183e02c6b475db3d4b0253ab5b90c, entries=150, sequenceid=430, filesize=12.0 K 2024-11-22T17:15:48,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/158dbdee15a84e979de079e4e71f552e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/158dbdee15a84e979de079e4e71f552e 2024-11-22T17:15:48,148 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/158dbdee15a84e979de079e4e71f552e, entries=150, sequenceid=430, filesize=12.0 K 2024-11-22T17:15:48,149 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 0da185830e822d28cb5b7cd0a28bc9f1 in 95ms, sequenceid=430, compaction requested=false 2024-11-22T17:15:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-22T17:15:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-22T17:15:48,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-22T17:15:48,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-11-22T17:15:48,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0220 sec 2024-11-22T17:15:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T17:15:48,272 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:48,273 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-22T17:15:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T17:15:48,276 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:48,277 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:48,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T17:15:48,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T17:15:48,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:48,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:48,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:48,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-22T17:15:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-22T17:15:48,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-22T17:15:48,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-22T17:15:48,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 162 msec 2024-11-22T17:15:48,490 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/84e95e9420a945daaae7dca289008d17 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/84e95e9420a945daaae7dca289008d17 2024-11-22T17:15:48,494 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/B of 0da185830e822d28cb5b7cd0a28bc9f1 into 84e95e9420a945daaae7dca289008d17(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:48,494 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:48,494 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/B, priority=13, startTime=1732295748043; duration=0sec 2024-11-22T17:15:48,494 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:48,495 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:B 2024-11-22T17:15:48,530 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/d98d22b095fc4262abb616bccc090096 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/d98d22b095fc4262abb616bccc090096 2024-11-22T17:15:48,537 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0da185830e822d28cb5b7cd0a28bc9f1/C of 0da185830e822d28cb5b7cd0a28bc9f1 into d98d22b095fc4262abb616bccc090096(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:48,537 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:48,538 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., storeName=0da185830e822d28cb5b7cd0a28bc9f1/C, priority=13, startTime=1732295748043; duration=0sec 2024-11-22T17:15:48,538 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:48,538 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0da185830e822d28cb5b7cd0a28bc9f1:C 2024-11-22T17:15:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T17:15:48,591 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:48,592 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-22T17:15:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T17:15:48,593 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:48,594 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:48,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T17:15:48,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T17:15:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 0da185830e822d28cb5b7cd0a28bc9f1: 2024-11-22T17:15:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:48,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-22T17:15:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-22T17:15:48,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-22T17:15:48,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-22T17:15:48,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 159 msec 2024-11-22T17:15:48,859 INFO [Thread-1008 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1008. 2024-11-22T17:15:48,859 INFO [Thread-1012 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1012. 2024-11-22T17:15:48,859 DEBUG [Thread-1008 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:48,859 DEBUG [Thread-1012 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:48,859 DEBUG [Thread-1012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,859 DEBUG [Thread-1008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,859 DEBUG [Thread-1008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:48,859 INFO [Thread-1004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1004. 2024-11-22T17:15:48,859 DEBUG [Thread-1004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:48,859 DEBUG [Thread-1012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,860 DEBUG [Thread-1004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:48,860 INFO [Thread-1010 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1010. 2024-11-22T17:15:48,860 DEBUG [Thread-1004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,860 DEBUG [Thread-1010 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:48,860 DEBUG [Thread-1010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:48,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:48,860 DEBUG [Thread-1010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,860 INFO [Thread-1006 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1006. 2024-11-22T17:15:48,860 DEBUG [Thread-1006 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:48,860 DEBUG [Thread-1006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,860 DEBUG [Thread-1006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T17:15:48,912 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:48,924 INFO [Thread-999 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-999. 2024-11-22T17:15:48,924 INFO [Thread-997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-997. 2024-11-22T17:15:48,924 INFO [Thread-995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-995. 2024-11-22T17:15:48,924 DEBUG [Thread-999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:48,924 DEBUG [Thread-997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:48,924 DEBUG [Thread-995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:48,924 DEBUG [Thread-999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,924 DEBUG [Thread-997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,924 DEBUG [Thread-995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,924 DEBUG [Thread-995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,924 DEBUG [Thread-997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,924 DEBUG [Thread-999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,924 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:48,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:48,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:48,944 INFO [Thread-1001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1001. 2024-11-22T17:15:48,945 DEBUG [Thread-1001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:48,945 DEBUG [Thread-1001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,945 DEBUG [Thread-1001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:48,945 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:51,145 INFO [Thread-993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-993. 2024-11-22T17:15:51,145 DEBUG [Thread-993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:15:51,145 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:51,146 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:51,146 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:51,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T17:15:51,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-22T17:15:51,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-22T17:15:51,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-22T17:15:51,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-22T17:15:51,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-22T17:15:51,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T17:15:51,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2826 2024-11-22T17:15:51,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2907 2024-11-22T17:15:51,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2907 2024-11-22T17:15:51,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2819 2024-11-22T17:15:51,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2914 2024-11-22T17:15:51,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T17:15:51,147 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T17:15:51,147 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T17:15:51,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:51,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:51,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:15:51,149 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T17:15:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T17:15:51,153 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295751153"}]},"ts":"1732295751153"} 2024-11-22T17:15:51,155 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T17:15:51,155 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T17:15:51,156 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T17:15:51,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0da185830e822d28cb5b7cd0a28bc9f1, UNASSIGN}] 2024-11-22T17:15:51,159 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0da185830e822d28cb5b7cd0a28bc9f1, UNASSIGN 2024-11-22T17:15:51,160 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=0da185830e822d28cb5b7cd0a28bc9f1, regionState=CLOSING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:51,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0da185830e822d28cb5b7cd0a28bc9f1, UNASSIGN because future has completed 2024-11-22T17:15:51,162 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T17:15:51,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:15:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T17:15:51,314 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:51,314 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T17:15:51,314 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 0da185830e822d28cb5b7cd0a28bc9f1, disabling compactions & flushes 2024-11-22T17:15:51,314 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:51,314 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:51,314 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. after waiting 0 ms 2024-11-22T17:15:51,314 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:51,314 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 0da185830e822d28cb5b7cd0a28bc9f1 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-22T17:15:51,315 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=A 2024-11-22T17:15:51,315 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:51,315 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=B 2024-11-22T17:15:51,315 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:51,315 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0da185830e822d28cb5b7cd0a28bc9f1, store=C 2024-11-22T17:15:51,315 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:51,319 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/855403f0812d481fb5a41dde2a5a2dd7 is 50, key is test_row_0/A:col10/1732295748921/Put/seqid=0 2024-11-22T17:15:51,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742117_1293 (size=12301) 2024-11-22T17:15:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T17:15:51,725 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/855403f0812d481fb5a41dde2a5a2dd7 2024-11-22T17:15:51,741 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/a9bf8248616a43d1ad18b618191eef3b is 50, key is test_row_0/B:col10/1732295748921/Put/seqid=0 2024-11-22T17:15:51,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742118_1294 (size=12301) 2024-11-22T17:15:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T17:15:52,146 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/a9bf8248616a43d1ad18b618191eef3b 2024-11-22T17:15:52,156 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/62a49016913646a6b0d3dd20fc243acc is 50, key is test_row_0/C:col10/1732295748921/Put/seqid=0 2024-11-22T17:15:52,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742119_1295 (size=12301) 2024-11-22T17:15:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T17:15:52,563 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/62a49016913646a6b0d3dd20fc243acc 2024-11-22T17:15:52,572 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/A/855403f0812d481fb5a41dde2a5a2dd7 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/855403f0812d481fb5a41dde2a5a2dd7 2024-11-22T17:15:52,579 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/855403f0812d481fb5a41dde2a5a2dd7, entries=150, sequenceid=443, filesize=12.0 K 2024-11-22T17:15:52,581 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/B/a9bf8248616a43d1ad18b618191eef3b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/a9bf8248616a43d1ad18b618191eef3b 2024-11-22T17:15:52,586 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/a9bf8248616a43d1ad18b618191eef3b, entries=150, sequenceid=443, filesize=12.0 K 2024-11-22T17:15:52,587 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/.tmp/C/62a49016913646a6b0d3dd20fc243acc as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/62a49016913646a6b0d3dd20fc243acc 2024-11-22T17:15:52,590 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/62a49016913646a6b0d3dd20fc243acc, entries=150, sequenceid=443, filesize=12.0 K 2024-11-22T17:15:52,591 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0da185830e822d28cb5b7cd0a28bc9f1 in 1277ms, sequenceid=443, compaction requested=true 2024-11-22T17:15:52,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bfd21899f715453ea1216b0a7959bd06, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/ad9e179afbd6496687b0ea24d2d5114c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f98b419bb5f043529178391af022082c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/de70e72c632d4918acd2986cc98b739b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/99e2ecaa39d14e6c896ca8c01fae9f84, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/45d0ab5c96ed4571847b0005c20af773, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e3650967e2c1418a99dc2134684215ca, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e8fb01058efb47268ae389ad0904b5c7, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f8b6ce8a731840fbb43ca542432f4d22, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/cb57942fc1c74274aa58563f16b4ebeb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/76c0eead5a0a4f51be14dfb0004984eb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bbb68560d49441398c9b19416e200588, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e82176bb410742c4b57339fc24bcc3ca, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/cce6608c22eb4bbfa27c02c2a6e2a9c7, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/d7ea55de53bc40509c92bd9691fde004, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f59f2d0df367428982c71e5e826ffe8b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/dc0ab77d2034489692a5b0ba39c6dba4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bc7b955de6684b5a8ee5506ae2e4b596, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/6a7bf0639e044ac7b0b0c135d9947258, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/85857bfdc5274d38abcd3b96e02b83c7, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/c409d50c2bfb47f9995b58ac7a6b71c5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/986d0a14d8404b4b80fc138026dccbbc, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/052aab980f6a4310a2a25607a1d1c63d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/565148225f15448eb79fcce6b3f3d5b8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/4f757080a6a1492487ff86c055397022, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/507b93fb26d044b1ace9968ed6f6a8e0] to archive 2024-11-22T17:15:52,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:15:52,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bfd21899f715453ea1216b0a7959bd06 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bfd21899f715453ea1216b0a7959bd06 2024-11-22T17:15:52,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/ad9e179afbd6496687b0ea24d2d5114c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/ad9e179afbd6496687b0ea24d2d5114c 2024-11-22T17:15:52,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f98b419bb5f043529178391af022082c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f98b419bb5f043529178391af022082c 2024-11-22T17:15:52,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/de70e72c632d4918acd2986cc98b739b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/de70e72c632d4918acd2986cc98b739b 2024-11-22T17:15:52,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/99e2ecaa39d14e6c896ca8c01fae9f84 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/99e2ecaa39d14e6c896ca8c01fae9f84 2024-11-22T17:15:52,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/45d0ab5c96ed4571847b0005c20af773 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/45d0ab5c96ed4571847b0005c20af773 2024-11-22T17:15:52,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e3650967e2c1418a99dc2134684215ca to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e3650967e2c1418a99dc2134684215ca 2024-11-22T17:15:52,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e8fb01058efb47268ae389ad0904b5c7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e8fb01058efb47268ae389ad0904b5c7 2024-11-22T17:15:52,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f8b6ce8a731840fbb43ca542432f4d22 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f8b6ce8a731840fbb43ca542432f4d22 2024-11-22T17:15:52,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/cb57942fc1c74274aa58563f16b4ebeb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/cb57942fc1c74274aa58563f16b4ebeb 2024-11-22T17:15:52,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/76c0eead5a0a4f51be14dfb0004984eb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/76c0eead5a0a4f51be14dfb0004984eb 2024-11-22T17:15:52,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bbb68560d49441398c9b19416e200588 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bbb68560d49441398c9b19416e200588 2024-11-22T17:15:52,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e82176bb410742c4b57339fc24bcc3ca to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/e82176bb410742c4b57339fc24bcc3ca 2024-11-22T17:15:52,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/cce6608c22eb4bbfa27c02c2a6e2a9c7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/cce6608c22eb4bbfa27c02c2a6e2a9c7 2024-11-22T17:15:52,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/d7ea55de53bc40509c92bd9691fde004 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/d7ea55de53bc40509c92bd9691fde004 2024-11-22T17:15:52,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f59f2d0df367428982c71e5e826ffe8b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/f59f2d0df367428982c71e5e826ffe8b 2024-11-22T17:15:52,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/dc0ab77d2034489692a5b0ba39c6dba4 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/dc0ab77d2034489692a5b0ba39c6dba4 2024-11-22T17:15:52,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bc7b955de6684b5a8ee5506ae2e4b596 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/bc7b955de6684b5a8ee5506ae2e4b596 2024-11-22T17:15:52,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/6a7bf0639e044ac7b0b0c135d9947258 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/6a7bf0639e044ac7b0b0c135d9947258 2024-11-22T17:15:52,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/85857bfdc5274d38abcd3b96e02b83c7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/85857bfdc5274d38abcd3b96e02b83c7 2024-11-22T17:15:52,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/c409d50c2bfb47f9995b58ac7a6b71c5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/c409d50c2bfb47f9995b58ac7a6b71c5 2024-11-22T17:15:52,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/986d0a14d8404b4b80fc138026dccbbc to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/986d0a14d8404b4b80fc138026dccbbc 2024-11-22T17:15:52,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/052aab980f6a4310a2a25607a1d1c63d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/052aab980f6a4310a2a25607a1d1c63d 2024-11-22T17:15:52,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/565148225f15448eb79fcce6b3f3d5b8 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/565148225f15448eb79fcce6b3f3d5b8 2024-11-22T17:15:52,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/4f757080a6a1492487ff86c055397022 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/4f757080a6a1492487ff86c055397022 2024-11-22T17:15:52,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/507b93fb26d044b1ace9968ed6f6a8e0 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/507b93fb26d044b1ace9968ed6f6a8e0 2024-11-22T17:15:52,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/a18ec83590ac432091ddbbee9dcda0f8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/7d6581e9b2484a2b825f9dc7fa7f52be, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/88caff994b274746979d3ff6993773ab, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/19bb3547648f4e2bbf8673126c77b1eb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/9e158601273e414ca1cf124e838539fa, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/83f7f61741914d3ab9889957963951af, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/3b0e2b3deaf741e1b9800c2ef106d275, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/60ddc34ac8fa43d6aae66d4f9cedb246, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/3d53b9a0b48648288af7861586844a42, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/28eb6b2bb7cc4789b4aba09ad236864c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/d6642a94dc0c4425a7bd769c881db16c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/708fd9206db947e984ea4732ac8e9f32, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/14601dec565346c29f4f28aa825fcc35, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/d2ffeaefa07c4b2c965a97df1e6812fd, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/377292ccb91e4240acd2af4affbe3bcb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/5d365cc6e9984711b5e04f31496c5691, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/af920fb7085f401a91f0a790a0ef94bb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/4d251e632e9b4fde809c787fbcd7d1ae, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/12c353d4d6574dcea106ea6599ba1630, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/91cc2d0b274b40a7b65c9500fb093ac3, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/c8dc4a666de44d2fa24c5a933b33e059, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/8236d451e36a468da39e03ab8444c4a6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/f6a567b4b3ef491a854192c6104535d2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/2e47c087cfa340edb97af4a4c820df46, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/c6078d8788eb4335b0a709eb7e42f048, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/b81a630115d84b65b9d743c0d882f97d] to archive 2024-11-22T17:15:52,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:15:52,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/a18ec83590ac432091ddbbee9dcda0f8 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/a18ec83590ac432091ddbbee9dcda0f8 2024-11-22T17:15:52,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/7d6581e9b2484a2b825f9dc7fa7f52be to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/7d6581e9b2484a2b825f9dc7fa7f52be 2024-11-22T17:15:52,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/88caff994b274746979d3ff6993773ab to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/88caff994b274746979d3ff6993773ab 2024-11-22T17:15:52,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/19bb3547648f4e2bbf8673126c77b1eb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/19bb3547648f4e2bbf8673126c77b1eb 2024-11-22T17:15:52,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/9e158601273e414ca1cf124e838539fa to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/9e158601273e414ca1cf124e838539fa 2024-11-22T17:15:52,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/83f7f61741914d3ab9889957963951af to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/83f7f61741914d3ab9889957963951af 2024-11-22T17:15:52,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/3b0e2b3deaf741e1b9800c2ef106d275 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/3b0e2b3deaf741e1b9800c2ef106d275 2024-11-22T17:15:52,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/60ddc34ac8fa43d6aae66d4f9cedb246 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/60ddc34ac8fa43d6aae66d4f9cedb246 2024-11-22T17:15:52,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/3d53b9a0b48648288af7861586844a42 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/3d53b9a0b48648288af7861586844a42 2024-11-22T17:15:52,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/28eb6b2bb7cc4789b4aba09ad236864c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/28eb6b2bb7cc4789b4aba09ad236864c 2024-11-22T17:15:52,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/d6642a94dc0c4425a7bd769c881db16c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/d6642a94dc0c4425a7bd769c881db16c 2024-11-22T17:15:52,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/708fd9206db947e984ea4732ac8e9f32 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/708fd9206db947e984ea4732ac8e9f32 2024-11-22T17:15:52,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/14601dec565346c29f4f28aa825fcc35 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/14601dec565346c29f4f28aa825fcc35 2024-11-22T17:15:52,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/d2ffeaefa07c4b2c965a97df1e6812fd to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/d2ffeaefa07c4b2c965a97df1e6812fd 2024-11-22T17:15:52,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/377292ccb91e4240acd2af4affbe3bcb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/377292ccb91e4240acd2af4affbe3bcb 2024-11-22T17:15:52,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/5d365cc6e9984711b5e04f31496c5691 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/5d365cc6e9984711b5e04f31496c5691 2024-11-22T17:15:52,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/af920fb7085f401a91f0a790a0ef94bb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/af920fb7085f401a91f0a790a0ef94bb 2024-11-22T17:15:52,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/4d251e632e9b4fde809c787fbcd7d1ae to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/4d251e632e9b4fde809c787fbcd7d1ae 2024-11-22T17:15:52,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/12c353d4d6574dcea106ea6599ba1630 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/12c353d4d6574dcea106ea6599ba1630 2024-11-22T17:15:52,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/91cc2d0b274b40a7b65c9500fb093ac3 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/91cc2d0b274b40a7b65c9500fb093ac3 2024-11-22T17:15:52,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/c8dc4a666de44d2fa24c5a933b33e059 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/c8dc4a666de44d2fa24c5a933b33e059 2024-11-22T17:15:52,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/8236d451e36a468da39e03ab8444c4a6 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/8236d451e36a468da39e03ab8444c4a6 2024-11-22T17:15:52,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/f6a567b4b3ef491a854192c6104535d2 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/f6a567b4b3ef491a854192c6104535d2 2024-11-22T17:15:52,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/2e47c087cfa340edb97af4a4c820df46 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/2e47c087cfa340edb97af4a4c820df46 2024-11-22T17:15:52,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/c6078d8788eb4335b0a709eb7e42f048 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/c6078d8788eb4335b0a709eb7e42f048 2024-11-22T17:15:52,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/b81a630115d84b65b9d743c0d882f97d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/b81a630115d84b65b9d743c0d882f97d 2024-11-22T17:15:52,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/be28215cbd0449c2898c3268c1fed432, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/2739783baa624217954e61293410c8b9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/754d826347e94adaac13bb6549823acb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/46d9e5c5304b4a6ebc34a97732eff88a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/068cba49975d4ffbb94ec36f4f95151a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/e96ee8c497ee40c2936b5990a5e377ef, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/5ca3a8c4520344398510a8ef11b8e54e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/603f6c2cbba745858dc262b69eeeda92, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/4056fe9be87e4fc58effa66438610fbe, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/0d135a8e433e4244821c19bd62ecb55a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/50c6aa246c174ba7aeba59e9c4fb04ae, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3bf007b356184a6fa0d0ca026d9b32b6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/41af56bc711a45288e70677017361203, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/57dbf55b029d458aa31ac69eca035cd2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/32d3dee3484b432c95aa9cce23db207c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/79950141c14340afb6b0f6e4dad18098, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3a7e6320ad9745caa1d79940a21c4184, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/72fcd3c4771340938e176261203af5a0, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/c26fef4dd1be4dab9dd9e56bfb292bf6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3933cb9cfeda4ab7a22e02815917a4b5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/e15ead38ca9944e89b0fd93e6dd34f9d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/0ca30c3c88ab4ee6a02e5f362b1fc5f6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/ac6e6941564b486fbf94c0a7952eedc8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/20d643644d6d4c8e9abf6c6da0c902f1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/09f3413a32e54c539e4e2d0b909966b0, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/a6373a11bb1d48539f8a0539df56b293] to archive 2024-11-22T17:15:52,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:15:52,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/be28215cbd0449c2898c3268c1fed432 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/be28215cbd0449c2898c3268c1fed432 2024-11-22T17:15:52,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/2739783baa624217954e61293410c8b9 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/2739783baa624217954e61293410c8b9 2024-11-22T17:15:52,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/754d826347e94adaac13bb6549823acb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/754d826347e94adaac13bb6549823acb 2024-11-22T17:15:52,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/46d9e5c5304b4a6ebc34a97732eff88a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/46d9e5c5304b4a6ebc34a97732eff88a 2024-11-22T17:15:52,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/068cba49975d4ffbb94ec36f4f95151a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/068cba49975d4ffbb94ec36f4f95151a 2024-11-22T17:15:52,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/e96ee8c497ee40c2936b5990a5e377ef to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/e96ee8c497ee40c2936b5990a5e377ef 2024-11-22T17:15:52,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/5ca3a8c4520344398510a8ef11b8e54e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/5ca3a8c4520344398510a8ef11b8e54e 2024-11-22T17:15:52,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/603f6c2cbba745858dc262b69eeeda92 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/603f6c2cbba745858dc262b69eeeda92 2024-11-22T17:15:52,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/4056fe9be87e4fc58effa66438610fbe to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/4056fe9be87e4fc58effa66438610fbe 2024-11-22T17:15:52,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/0d135a8e433e4244821c19bd62ecb55a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/0d135a8e433e4244821c19bd62ecb55a 2024-11-22T17:15:52,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/50c6aa246c174ba7aeba59e9c4fb04ae to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/50c6aa246c174ba7aeba59e9c4fb04ae 2024-11-22T17:15:52,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3bf007b356184a6fa0d0ca026d9b32b6 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3bf007b356184a6fa0d0ca026d9b32b6 2024-11-22T17:15:52,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/41af56bc711a45288e70677017361203 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/41af56bc711a45288e70677017361203 2024-11-22T17:15:52,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/57dbf55b029d458aa31ac69eca035cd2 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/57dbf55b029d458aa31ac69eca035cd2 2024-11-22T17:15:52,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/32d3dee3484b432c95aa9cce23db207c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/32d3dee3484b432c95aa9cce23db207c 2024-11-22T17:15:52,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/79950141c14340afb6b0f6e4dad18098 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/79950141c14340afb6b0f6e4dad18098 2024-11-22T17:15:52,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3a7e6320ad9745caa1d79940a21c4184 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3a7e6320ad9745caa1d79940a21c4184 2024-11-22T17:15:52,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/72fcd3c4771340938e176261203af5a0 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/72fcd3c4771340938e176261203af5a0 2024-11-22T17:15:52,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/c26fef4dd1be4dab9dd9e56bfb292bf6 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/c26fef4dd1be4dab9dd9e56bfb292bf6 2024-11-22T17:15:52,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3933cb9cfeda4ab7a22e02815917a4b5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/3933cb9cfeda4ab7a22e02815917a4b5 2024-11-22T17:15:52,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/e15ead38ca9944e89b0fd93e6dd34f9d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/e15ead38ca9944e89b0fd93e6dd34f9d 2024-11-22T17:15:52,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/0ca30c3c88ab4ee6a02e5f362b1fc5f6 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/0ca30c3c88ab4ee6a02e5f362b1fc5f6 2024-11-22T17:15:52,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/ac6e6941564b486fbf94c0a7952eedc8 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/ac6e6941564b486fbf94c0a7952eedc8 2024-11-22T17:15:52,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/20d643644d6d4c8e9abf6c6da0c902f1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/20d643644d6d4c8e9abf6c6da0c902f1 2024-11-22T17:15:52,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/09f3413a32e54c539e4e2d0b909966b0 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/09f3413a32e54c539e4e2d0b909966b0 2024-11-22T17:15:52,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/a6373a11bb1d48539f8a0539df56b293 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/a6373a11bb1d48539f8a0539df56b293 2024-11-22T17:15:52,664 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/recovered.edits/446.seqid, newMaxSeqId=446, maxSeqId=1 2024-11-22T17:15:52,665 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. 2024-11-22T17:15:52,665 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 0da185830e822d28cb5b7cd0a28bc9f1: Waiting for close lock at 1732295751314Running coprocessor pre-close hooks at 1732295751314Disabling compacts and flushes for region at 1732295751314Disabling writes for close at 1732295751314Obtaining lock to block concurrent updates at 1732295751314Preparing flush snapshotting stores in 0da185830e822d28cb5b7cd0a28bc9f1 at 1732295751314Finished memstore snapshotting TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732295751315 (+1 ms)Flushing stores of TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1. at 1732295751315Flushing 0da185830e822d28cb5b7cd0a28bc9f1/A: creating writer at 1732295751315Flushing 0da185830e822d28cb5b7cd0a28bc9f1/A: appending metadata at 1732295751319 (+4 ms)Flushing 0da185830e822d28cb5b7cd0a28bc9f1/A: closing flushed file at 1732295751319Flushing 0da185830e822d28cb5b7cd0a28bc9f1/B: creating writer at 1732295751735 (+416 ms)Flushing 0da185830e822d28cb5b7cd0a28bc9f1/B: appending metadata at 1732295751740 (+5 ms)Flushing 0da185830e822d28cb5b7cd0a28bc9f1/B: closing flushed file at 1732295751740Flushing 0da185830e822d28cb5b7cd0a28bc9f1/C: creating writer at 1732295752151 (+411 ms)Flushing 0da185830e822d28cb5b7cd0a28bc9f1/C: appending metadata at 1732295752155 (+4 ms)Flushing 0da185830e822d28cb5b7cd0a28bc9f1/C: closing flushed file at 1732295752155Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5db80c71: reopening flushed file at 1732295752571 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a85cef6: reopening flushed file at 1732295752580 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49e1d05d: reopening flushed file at 1732295752586 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0da185830e822d28cb5b7cd0a28bc9f1 in 1277ms, sequenceid=443, compaction requested=true at 1732295752591 (+5 ms)Writing region close event to WAL at 1732295752662 (+71 ms)Running coprocessor post-close hooks at 1732295752664 (+2 ms)Closed at 1732295752665 (+1 ms) 2024-11-22T17:15:52,667 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:52,667 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=0da185830e822d28cb5b7cd0a28bc9f1, regionState=CLOSED 2024-11-22T17:15:52,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:15:52,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-22T17:15:52,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 0da185830e822d28cb5b7cd0a28bc9f1, server=e61df7af61d2,42253,1732295655623 in 1.5070 sec 2024-11-22T17:15:52,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-22T17:15:52,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0da185830e822d28cb5b7cd0a28bc9f1, UNASSIGN in 1.5130 sec 2024-11-22T17:15:52,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-22T17:15:52,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5180 sec 2024-11-22T17:15:52,678 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295752677"}]},"ts":"1732295752677"} 2024-11-22T17:15:52,679 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T17:15:52,679 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T17:15:52,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5310 sec 2024-11-22T17:15:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T17:15:53,301 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:53,302 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T17:15:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:53,304 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:53,304 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T17:15:53,306 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:53,308 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/recovered.edits] 2024-11-22T17:15:53,311 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/855403f0812d481fb5a41dde2a5a2dd7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/855403f0812d481fb5a41dde2a5a2dd7 2024-11-22T17:15:53,312 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/d5944463f8fa41e5be5b543ae92e9f2c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/d5944463f8fa41e5be5b543ae92e9f2c 2024-11-22T17:15:53,313 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/eecc53ba57274e45b88c99b51c708ee5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/A/eecc53ba57274e45b88c99b51c708ee5 2024-11-22T17:15:53,315 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/01a183e02c6b475db3d4b0253ab5b90c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/01a183e02c6b475db3d4b0253ab5b90c 2024-11-22T17:15:53,316 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/84e95e9420a945daaae7dca289008d17 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/84e95e9420a945daaae7dca289008d17 2024-11-22T17:15:53,317 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/a9bf8248616a43d1ad18b618191eef3b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/B/a9bf8248616a43d1ad18b618191eef3b 2024-11-22T17:15:53,318 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/158dbdee15a84e979de079e4e71f552e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/158dbdee15a84e979de079e4e71f552e 2024-11-22T17:15:53,319 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/62a49016913646a6b0d3dd20fc243acc to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/62a49016913646a6b0d3dd20fc243acc 2024-11-22T17:15:53,320 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/d98d22b095fc4262abb616bccc090096 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/C/d98d22b095fc4262abb616bccc090096 2024-11-22T17:15:53,322 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/recovered.edits/446.seqid to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1/recovered.edits/446.seqid 2024-11-22T17:15:53,322 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/0da185830e822d28cb5b7cd0a28bc9f1 2024-11-22T17:15:53,322 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T17:15:53,324 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:53,325 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T17:15:53,327 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T17:15:53,328 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:53,328 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T17:15:53,328 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732295753328"}]},"ts":"9223372036854775807"} 2024-11-22T17:15:53,330 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T17:15:53,330 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0da185830e822d28cb5b7cd0a28bc9f1, NAME => 'TestAcidGuarantees,,1732295726651.0da185830e822d28cb5b7cd0a28bc9f1.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T17:15:53,330 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T17:15:53,330 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732295753330"}]},"ts":"9223372036854775807"} 2024-11-22T17:15:53,332 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T17:15:53,333 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:53,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-11-22T17:15:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T17:15:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T17:15:53,412 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:53,420 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_659160475_22 at /127.0.0.1:45672 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 315) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2260 (was 3027) 2024-11-22T17:15:53,428 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=2259 2024-11-22T17:15:53,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T17:15:53,429 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T17:15:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:53,431 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T17:15:53,431 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:53,431 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-22T17:15:53,432 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T17:15:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-22T17:15:53,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742120_1296 (size=1061) 2024-11-22T17:15:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-22T17:15:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-22T17:15:53,840 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8 2024-11-22T17:15:53,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742121_1297 (size=53) 2024-11-22T17:15:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-22T17:15:54,252 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:15:54,252 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2477a80617ad0e4131ef1992c2af963e, disabling compactions & flushes 2024-11-22T17:15:54,252 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:54,252 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:54,252 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. after waiting 0 ms 2024-11-22T17:15:54,252 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:54,252 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:54,252 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2477a80617ad0e4131ef1992c2af963e: Waiting for close lock at 1732295754252Disabling compacts and flushes for region at 1732295754252Disabling writes for close at 1732295754252Writing region close event to WAL at 1732295754252Closed at 1732295754252 2024-11-22T17:15:54,254 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T17:15:54,254 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732295754254"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732295754254"}]},"ts":"1732295754254"} 2024-11-22T17:15:54,256 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T17:15:54,257 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T17:15:54,257 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295754257"}]},"ts":"1732295754257"} 2024-11-22T17:15:54,259 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T17:15:54,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, ASSIGN}] 2024-11-22T17:15:54,261 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, ASSIGN 2024-11-22T17:15:54,262 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, ASSIGN; state=OFFLINE, location=e61df7af61d2,42253,1732295655623; forceNewPlan=false, retain=false 2024-11-22T17:15:54,413 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=2477a80617ad0e4131ef1992c2af963e, regionState=OPENING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:54,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, ASSIGN because future has completed 2024-11-22T17:15:54,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:15:54,571 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:54,571 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} 2024-11-22T17:15:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-22T17:15:54,572 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:54,572 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:15:54,572 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:54,572 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:54,574 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:54,575 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:15:54,576 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2477a80617ad0e4131ef1992c2af963e columnFamilyName A 2024-11-22T17:15:54,576 DEBUG [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:54,577 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.HStore(327): Store=2477a80617ad0e4131ef1992c2af963e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:15:54,577 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:54,580 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:15:54,580 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2477a80617ad0e4131ef1992c2af963e columnFamilyName B 2024-11-22T17:15:54,580 DEBUG [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:54,581 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.HStore(327): Store=2477a80617ad0e4131ef1992c2af963e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:15:54,581 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:54,583 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:15:54,583 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2477a80617ad0e4131ef1992c2af963e columnFamilyName C 2024-11-22T17:15:54,583 DEBUG [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:54,584 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.HStore(327): Store=2477a80617ad0e4131ef1992c2af963e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:15:54,584 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:54,584 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:54,585 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:54,585 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:54,586 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:54,586 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:54,587 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T17:15:54,588 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:54,596 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T17:15:54,596 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 2477a80617ad0e4131ef1992c2af963e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71698691, jitterRate=0.0683937519788742}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T17:15:54,596 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:54,597 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 2477a80617ad0e4131ef1992c2af963e: Running coprocessor pre-open hook at 1732295754572Writing region info on filesystem at 1732295754572Initializing all the Stores at 1732295754573 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295754573Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295754574 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295754574Cleaning up temporary data from old regions at 1732295754586 (+12 ms)Running coprocessor post-open hooks at 1732295754596 (+10 ms)Region opened successfully at 1732295754597 (+1 ms) 2024-11-22T17:15:54,598 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., pid=89, masterSystemTime=1732295754567 2024-11-22T17:15:54,601 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:54,601 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:54,602 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=2477a80617ad0e4131ef1992c2af963e, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:54,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:15:54,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-22T17:15:54,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 in 194 msec 2024-11-22T17:15:54,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-22T17:15:54,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, ASSIGN in 356 msec 2024-11-22T17:15:54,621 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T17:15:54,621 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295754621"}]},"ts":"1732295754621"} 2024-11-22T17:15:54,625 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T17:15:54,626 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T17:15:54,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1970 sec 2024-11-22T17:15:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-22T17:15:55,582 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:55,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d493646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:55,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:55,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:55,586 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a2e459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:55,588 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cadf21e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:55,589 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:55,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:55,591 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T17:15:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e61df7af61d2,45693,1732295654407 2024-11-22T17:15:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6185740f 2024-11-22T17:15:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T17:15:55,593 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T17:15:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T17:15:55,594 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T17:15:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T17:15:55,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742122_1298 (size=1097) 2024-11-22T17:15:56,011 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-22T17:15:56,011 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-22T17:15:56,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T17:15:56,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, REOPEN/MOVE}] 2024-11-22T17:15:56,020 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, REOPEN/MOVE 2024-11-22T17:15:56,021 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=2477a80617ad0e4131ef1992c2af963e, regionState=CLOSING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:56,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, REOPEN/MOVE because future has completed 2024-11-22T17:15:56,024 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T17:15:56,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:15:56,177 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,178 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T17:15:56,178 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 2477a80617ad0e4131ef1992c2af963e, disabling compactions & flushes 2024-11-22T17:15:56,178 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:56,178 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:56,178 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. after waiting 0 ms 2024-11-22T17:15:56,178 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:56,182 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T17:15:56,183 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:56,183 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 2477a80617ad0e4131ef1992c2af963e: Waiting for close lock at 1732295756178Running coprocessor pre-close hooks at 1732295756178Disabling compacts and flushes for region at 1732295756178Disabling writes for close at 1732295756178Writing region close event to WAL at 1732295756178Running coprocessor post-close hooks at 1732295756182 (+4 ms)Closed at 1732295756182 2024-11-22T17:15:56,183 WARN [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 2477a80617ad0e4131ef1992c2af963e to self. 2024-11-22T17:15:56,185 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,185 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=2477a80617ad0e4131ef1992c2af963e, regionState=CLOSED 2024-11-22T17:15:56,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:15:56,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-22T17:15:56,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 in 163 msec 2024-11-22T17:15:56,189 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, REOPEN/MOVE; state=CLOSED, location=e61df7af61d2,42253,1732295655623; forceNewPlan=false, retain=true 2024-11-22T17:15:56,340 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=2477a80617ad0e4131ef1992c2af963e, regionState=OPENING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:56,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, REOPEN/MOVE because future has completed 2024-11-22T17:15:56,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:15:56,496 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:56,496 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} 2024-11-22T17:15:56,496 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,496 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:15:56,497 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,497 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,498 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,499 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:15:56,499 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2477a80617ad0e4131ef1992c2af963e columnFamilyName A 2024-11-22T17:15:56,500 DEBUG [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:56,500 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.HStore(327): Store=2477a80617ad0e4131ef1992c2af963e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:15:56,501 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,502 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:15:56,502 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2477a80617ad0e4131ef1992c2af963e columnFamilyName B 2024-11-22T17:15:56,502 DEBUG [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:56,502 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.HStore(327): Store=2477a80617ad0e4131ef1992c2af963e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:15:56,503 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,503 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:15:56,503 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2477a80617ad0e4131ef1992c2af963e columnFamilyName C 2024-11-22T17:15:56,503 DEBUG [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:56,504 INFO [StoreOpener-2477a80617ad0e4131ef1992c2af963e-1 {}] regionserver.HStore(327): Store=2477a80617ad0e4131ef1992c2af963e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:15:56,504 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:56,504 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,505 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,506 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,506 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,506 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,507 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T17:15:56,508 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,509 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 2477a80617ad0e4131ef1992c2af963e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70968202, jitterRate=0.057508617639541626}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T17:15:56,509 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,510 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 2477a80617ad0e4131ef1992c2af963e: Running coprocessor pre-open hook at 1732295756497Writing region info on filesystem at 1732295756497Initializing all the Stores at 1732295756497Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295756497Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295756498 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295756498Cleaning up temporary data from old regions at 1732295756506 (+8 ms)Running coprocessor post-open hooks at 1732295756509 (+3 ms)Region opened successfully at 1732295756510 (+1 ms) 2024-11-22T17:15:56,510 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., pid=94, masterSystemTime=1732295756493 2024-11-22T17:15:56,512 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:56,512 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:56,513 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=2477a80617ad0e4131ef1992c2af963e, regionState=OPEN, openSeqNum=5, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:56,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:15:56,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-22T17:15:56,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 in 174 msec 2024-11-22T17:15:56,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-22T17:15:56,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, REOPEN/MOVE in 498 msec 2024-11-22T17:15:56,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-22T17:15:56,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-11-22T17:15:56,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 927 msec 2024-11-22T17:15:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-22T17:15:56,524 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:56,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c6bde19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:56,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:56,526 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:56,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71629ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:56,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:56,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:56,528 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325eb487, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173aa1ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:56,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:56,530 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:56,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:56,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e56f298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:56,532 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:56,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb31df0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5e8400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:56,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:56,534 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59f140f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:56,535 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:56,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dfb6b54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce5c6bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:56,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:56,537 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:56,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:56,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3416d4d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:56,539 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:56,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c96ab2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7fbdbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:56,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:56,541 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@795f82bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:56,543 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:56,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f178f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69edc398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:56,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:56,545 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41b2493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:56,546 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c2f2f10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbf44a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:56,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:56,549 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3516609b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:56,550 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4786b710, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ad32ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:56,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:56,552 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:56,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:56,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:56,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30fed636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:56,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:56,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:56,554 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:56,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47794889, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615de503, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:56,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:56,555 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b332893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:56,557 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:56,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf442bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5690e3aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:15:56,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:15:56,559 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:15:56,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:15:56,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:15:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51ea01c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:15:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:15:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:15:56,561 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:15:56,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec67c57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:15:56,562 DEBUG [Thread-1401 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:56,562 DEBUG [Thread-1395 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:56,562 DEBUG [Thread-1399 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:56,562 DEBUG [Thread-1397 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:56,564 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:56,564 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:56,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:56,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:15:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:15:56,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:15:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:15:56,573 DEBUG [Thread-1382 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:56,573 DEBUG [Thread-1386 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:56,574 DEBUG [Thread-1388 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:56,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:56,576 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:15:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:15:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:56,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:56,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:56,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:15:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-22T17:15:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-22T17:15:56,584 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:56,584 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:56,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:56,585 DEBUG [Thread-1393 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:56,586 DEBUG [Thread-1393 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:56,586 DEBUG [Thread-1393 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:56,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:56,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:15:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,593 DEBUG [Thread-1390 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:56,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:15:56,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:15:56,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:56,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:15:56,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:56,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:15:56,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:56,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:15:56,598 DEBUG [Thread-1384 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:15:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:15:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:15:56,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:15:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:15:56,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226fbd56000d9448b3834541c63901831e_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295756590/Put/seqid=0 2024-11-22T17:15:56,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742123_1299 (size=12154) 2024-11-22T17:15:56,627 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:56,630 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226fbd56000d9448b3834541c63901831e_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226fbd56000d9448b3834541c63901831e_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:56,632 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/ab1f39e74bd54456a1f259fcdcf6e61c, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:56,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/ab1f39e74bd54456a1f259fcdcf6e61c is 175, key is test_row_0/A:col10/1732295756590/Put/seqid=0 2024-11-22T17:15:56,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295756616, max timestamp=1732295756630Num uniques 859; 2024-11-22T17:15:56,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295756616, max timestamp=1732295756630Num uniques 859; 2024-11-22T17:15:56,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:56,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:56,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295756616, max timestamp=1732295756630Num uniques 859; 2024-11-22T17:15:56,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:56,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742124_1300 (size=30951) 2024-11-22T17:15:56,644 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/ab1f39e74bd54456a1f259fcdcf6e61c 2024-11-22T17:15:56,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/d55ad39b211f4ed7b2687f3eefd799b1 is 50, key is test_row_0/B:col10/1732295756590/Put/seqid=0 2024-11-22T17:15:56,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742125_1301 (size=9657) 2024-11-22T17:15:56,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:56,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295816667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295816665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295816667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:56,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295816668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295816668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:56,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-22T17:15:56,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-22T17:15:56,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:56,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:15:56,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:56,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:56,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:56,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295816782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:56,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295816791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295816791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:56,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295816791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295816791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:56,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:56,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:56,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-22T17:15:56,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:56,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:15:56,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:56,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:56,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-22T17:15:57,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295817001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295817001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295817002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295817002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295817002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-22T17:15:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:15:57,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:57,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:57,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:57,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/d55ad39b211f4ed7b2687f3eefd799b1 2024-11-22T17:15:57,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/345ee4b0a3274c809f4fec557329dbf8 is 50, key is test_row_0/C:col10/1732295756590/Put/seqid=0 2024-11-22T17:15:57,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742126_1302 (size=9657) 2024-11-22T17:15:57,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/345ee4b0a3274c809f4fec557329dbf8 2024-11-22T17:15:57,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/ab1f39e74bd54456a1f259fcdcf6e61c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/ab1f39e74bd54456a1f259fcdcf6e61c 2024-11-22T17:15:57,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/ab1f39e74bd54456a1f259fcdcf6e61c, entries=150, sequenceid=14, filesize=30.2 K 2024-11-22T17:15:57,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/d55ad39b211f4ed7b2687f3eefd799b1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/d55ad39b211f4ed7b2687f3eefd799b1 2024-11-22T17:15:57,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/d55ad39b211f4ed7b2687f3eefd799b1, entries=100, sequenceid=14, filesize=9.4 K 2024-11-22T17:15:57,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/345ee4b0a3274c809f4fec557329dbf8 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/345ee4b0a3274c809f4fec557329dbf8 2024-11-22T17:15:57,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/345ee4b0a3274c809f4fec557329dbf8, entries=100, sequenceid=14, filesize=9.4 K 2024-11-22T17:15:57,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 2477a80617ad0e4131ef1992c2af963e in 577ms, sequenceid=14, compaction requested=false 2024-11-22T17:15:57,170 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T17:15:57,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:15:57,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-22T17:15:57,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:57,202 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-22T17:15:57,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:15:57,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:15:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:15:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-22T17:15:57,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122959a11dcb9d24fce9e95764599f1105f_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295756667/Put/seqid=0 2024-11-22T17:15:57,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742127_1303 (size=12154) 2024-11-22T17:15:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,276 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122959a11dcb9d24fce9e95764599f1105f_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122959a11dcb9d24fce9e95764599f1105f_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:57,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5dfbab3d139948f6b66d774f3780a180, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:57,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5dfbab3d139948f6b66d774f3780a180 is 175, key is test_row_0/A:col10/1732295756667/Put/seqid=0 2024-11-22T17:15:57,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742128_1304 (size=30955) 2024-11-22T17:15:57,306 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5dfbab3d139948f6b66d774f3780a180 2024-11-22T17:15:57,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/f3bd8eaa6e724f0eaf260512d1d3866e is 50, key is test_row_0/B:col10/1732295756667/Put/seqid=0 2024-11-22T17:15:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:57,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:15:57,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742129_1305 (size=12001) 2024-11-22T17:15:57,350 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/f3bd8eaa6e724f0eaf260512d1d3866e 2024-11-22T17:15:57,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/cd0e787ad4d54e2985ba86d43593d82c is 50, key is test_row_0/C:col10/1732295756667/Put/seqid=0 2024-11-22T17:15:57,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295817343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295817343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742130_1306 (size=12001) 2024-11-22T17:15:57,375 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/cd0e787ad4d54e2985ba86d43593d82c 2024-11-22T17:15:57,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5dfbab3d139948f6b66d774f3780a180 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5dfbab3d139948f6b66d774f3780a180 2024-11-22T17:15:57,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295817364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295817366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295817366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,388 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5dfbab3d139948f6b66d774f3780a180, entries=150, sequenceid=51, filesize=30.2 K 2024-11-22T17:15:57,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/f3bd8eaa6e724f0eaf260512d1d3866e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/f3bd8eaa6e724f0eaf260512d1d3866e 2024-11-22T17:15:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,394 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/f3bd8eaa6e724f0eaf260512d1d3866e, entries=150, sequenceid=51, filesize=11.7 K 2024-11-22T17:15:57,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/cd0e787ad4d54e2985ba86d43593d82c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/cd0e787ad4d54e2985ba86d43593d82c 2024-11-22T17:15:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,405 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/cd0e787ad4d54e2985ba86d43593d82c, entries=150, sequenceid=51, filesize=11.7 K 2024-11-22T17:15:57,406 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=53.67 KB/54960 for 2477a80617ad0e4131ef1992c2af963e in 204ms, sequenceid=51, compaction requested=false 2024-11-22T17:15:57,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:15:57,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:57,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-22T17:15:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-22T17:15:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-22T17:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 824 msec 2024-11-22T17:15:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 829 msec 2024-11-22T17:15:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:57,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T17:15:57,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:15:57,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:57,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:15:57,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:57,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:15:57,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122eb46bb19c0cd4e1b8c83e9138af18f86_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295757478/Put/seqid=0 2024-11-22T17:15:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742131_1307 (size=12154) 2024-11-22T17:15:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,500 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,509 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122eb46bb19c0cd4e1b8c83e9138af18f86_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122eb46bb19c0cd4e1b8c83e9138af18f86_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,512 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/6130a0dfc40447c1b51096374049e07f, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/6130a0dfc40447c1b51096374049e07f is 175, key is test_row_0/A:col10/1732295757478/Put/seqid=0 2024-11-22T17:15:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:57,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742132_1308 (size=30955) 2024-11-22T17:15:57,529 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/6130a0dfc40447c1b51096374049e07f 2024-11-22T17:15:57,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/0152c1a305624d61acc08669ba4a46ca is 50, key is test_row_0/B:col10/1732295757478/Put/seqid=0 2024-11-22T17:15:57,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742133_1309 (size=12001) 2024-11-22T17:15:57,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295817558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295817559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295817559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295817560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295817561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295817671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295817682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295817682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295817682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295817682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-22T17:15:57,722 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:57,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-22T17:15:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-22T17:15:57,724 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:57,724 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:57,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-22T17:15:57,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-22T17:15:57,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:57,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:15:57,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:57,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:57,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:57,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295817891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295817892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295817892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295817892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295817892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:57,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/0152c1a305624d61acc08669ba4a46ca 2024-11-22T17:15:57,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/a0eb1106e0de41f1b7895174210b2fd3 is 50, key is test_row_0/C:col10/1732295757478/Put/seqid=0 2024-11-22T17:15:57,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742134_1310 (size=12001) 2024-11-22T17:15:58,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-22T17:15:58,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:58,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:15:58,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:58,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:58,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:58,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-22T17:15:58,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-22T17:15:58,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:58,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:15:58,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:58,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:58,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:58,201 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T17:15:58,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295818212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295818213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295818214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295818214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295818214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-22T17:15:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:15:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:58,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:15:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-22T17:15:58,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/a0eb1106e0de41f1b7895174210b2fd3 2024-11-22T17:15:58,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/6130a0dfc40447c1b51096374049e07f as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/6130a0dfc40447c1b51096374049e07f 2024-11-22T17:15:58,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/6130a0dfc40447c1b51096374049e07f, entries=150, sequenceid=63, filesize=30.2 K 2024-11-22T17:15:58,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/0152c1a305624d61acc08669ba4a46ca as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/0152c1a305624d61acc08669ba4a46ca 2024-11-22T17:15:58,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/0152c1a305624d61acc08669ba4a46ca, entries=150, sequenceid=63, filesize=11.7 K 2024-11-22T17:15:58,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/a0eb1106e0de41f1b7895174210b2fd3 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/a0eb1106e0de41f1b7895174210b2fd3 2024-11-22T17:15:58,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/a0eb1106e0de41f1b7895174210b2fd3, entries=150, sequenceid=63, filesize=11.7 K 2024-11-22T17:15:58,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2477a80617ad0e4131ef1992c2af963e in 928ms, sequenceid=63, compaction requested=true 2024-11-22T17:15:58,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:15:58,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:15:58,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:58,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:15:58,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T17:15:58,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:15:58,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-22T17:15:58,409 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:58,409 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:58,413 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:58,413 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/C is initiating minor compaction (all files) 2024-11-22T17:15:58,413 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/C in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:58,413 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/345ee4b0a3274c809f4fec557329dbf8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/cd0e787ad4d54e2985ba86d43593d82c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/a0eb1106e0de41f1b7895174210b2fd3] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=32.9 K 2024-11-22T17:15:58,414 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:58,414 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/A is initiating minor compaction (all files) 2024-11-22T17:15:58,414 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 345ee4b0a3274c809f4fec557329dbf8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732295756583 2024-11-22T17:15:58,418 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting cd0e787ad4d54e2985ba86d43593d82c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732295756665 2024-11-22T17:15:58,418 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a0eb1106e0de41f1b7895174210b2fd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732295757338 2024-11-22T17:15:58,432 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/A in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:58,433 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/ab1f39e74bd54456a1f259fcdcf6e61c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5dfbab3d139948f6b66d774f3780a180, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/6130a0dfc40447c1b51096374049e07f] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=90.7 K 2024-11-22T17:15:58,433 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:58,433 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/ab1f39e74bd54456a1f259fcdcf6e61c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5dfbab3d139948f6b66d774f3780a180, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/6130a0dfc40447c1b51096374049e07f] 2024-11-22T17:15:58,435 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab1f39e74bd54456a1f259fcdcf6e61c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732295756583 2024-11-22T17:15:58,436 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dfbab3d139948f6b66d774f3780a180, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732295756665 2024-11-22T17:15:58,437 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6130a0dfc40447c1b51096374049e07f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732295757338 2024-11-22T17:15:58,451 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#C#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:58,451 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/157e182ebcda43df97e80c5c814e06ed is 50, key is test_row_0/C:col10/1732295757478/Put/seqid=0 2024-11-22T17:15:58,475 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:58,494 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411223ca08772240a4cd0bea0d3fd453ac592_2477a80617ad0e4131ef1992c2af963e store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:58,496 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411223ca08772240a4cd0bea0d3fd453ac592_2477a80617ad0e4131ef1992c2af963e, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:58,496 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223ca08772240a4cd0bea0d3fd453ac592_2477a80617ad0e4131ef1992c2af963e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:58,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-22T17:15:58,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:58,497 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T17:15:58,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:15:58,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:58,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:15:58,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:58,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:15:58,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:58,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742135_1311 (size=12104) 2024-11-22T17:15:58,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742136_1312 (size=4469) 2024-11-22T17:15:58,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122476ef6d4e0a5410d9b1b3772c28fbe28_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295757560/Put/seqid=0 2024-11-22T17:15:58,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742137_1313 (size=12154) 2024-11-22T17:15:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:58,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:15:58,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295818756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295818760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295818760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295818760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295818760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-22T17:15:58,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295818872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295818872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295818872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295818872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295818873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:58,933 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/157e182ebcda43df97e80c5c814e06ed as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/157e182ebcda43df97e80c5c814e06ed 2024-11-22T17:15:58,939 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/C of 2477a80617ad0e4131ef1992c2af963e into 157e182ebcda43df97e80c5c814e06ed(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:58,939 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:15:58,939 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/C, priority=13, startTime=1732295758409; duration=0sec 2024-11-22T17:15:58,940 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:58,940 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:C 2024-11-22T17:15:58,940 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:58,942 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:58,943 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/B is initiating minor compaction (all files) 2024-11-22T17:15:58,943 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/B in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:58,943 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/d55ad39b211f4ed7b2687f3eefd799b1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/f3bd8eaa6e724f0eaf260512d1d3866e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/0152c1a305624d61acc08669ba4a46ca] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=32.9 K 2024-11-22T17:15:58,943 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting d55ad39b211f4ed7b2687f3eefd799b1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732295756583 2024-11-22T17:15:58,944 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting f3bd8eaa6e724f0eaf260512d1d3866e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732295756665 2024-11-22T17:15:58,944 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 0152c1a305624d61acc08669ba4a46ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732295757338 2024-11-22T17:15:58,966 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:58,967 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/8e3de9dd7d7540139c6342c11c32bc12 is 50, key is test_row_0/B:col10/1732295757478/Put/seqid=0 2024-11-22T17:15:58,974 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#A#compaction#265 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:58,975 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/64c28e0a96ce4e24af031e0295d221c1 is 175, key is test_row_0/A:col10/1732295757478/Put/seqid=0 2024-11-22T17:15:58,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742138_1314 (size=12104) 2024-11-22T17:15:59,003 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/8e3de9dd7d7540139c6342c11c32bc12 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8e3de9dd7d7540139c6342c11c32bc12 2024-11-22T17:15:59,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742139_1315 (size=31058) 2024-11-22T17:15:59,019 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/B of 2477a80617ad0e4131ef1992c2af963e into 8e3de9dd7d7540139c6342c11c32bc12(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:59,019 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:15:59,019 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/B, priority=13, startTime=1732295758409; duration=0sec 2024-11-22T17:15:59,021 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:59,021 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:B 2024-11-22T17:15:59,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:59,049 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122476ef6d4e0a5410d9b1b3772c28fbe28_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122476ef6d4e0a5410d9b1b3772c28fbe28_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:59,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/1046d1bd1317455e8795efa4d7f60818, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:59,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/1046d1bd1317455e8795efa4d7f60818 is 175, key is test_row_0/A:col10/1732295757560/Put/seqid=0 2024-11-22T17:15:59,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295819092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295819092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295819092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295819092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295819092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742140_1316 (size=30955) 2024-11-22T17:15:59,111 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/1046d1bd1317455e8795efa4d7f60818 2024-11-22T17:15:59,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/ccae1428ef6a490680707cb74caed610 is 50, key is test_row_0/B:col10/1732295757560/Put/seqid=0 2024-11-22T17:15:59,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742141_1317 (size=12001) 2024-11-22T17:15:59,186 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/ccae1428ef6a490680707cb74caed610 2024-11-22T17:15:59,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/21efe3e3476944d6b3e44fd057028e1d is 50, key is test_row_0/C:col10/1732295757560/Put/seqid=0 2024-11-22T17:15:59,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742142_1318 (size=12001) 2024-11-22T17:15:59,251 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/21efe3e3476944d6b3e44fd057028e1d 2024-11-22T17:15:59,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/1046d1bd1317455e8795efa4d7f60818 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/1046d1bd1317455e8795efa4d7f60818 2024-11-22T17:15:59,271 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/1046d1bd1317455e8795efa4d7f60818, entries=150, sequenceid=83, filesize=30.2 K 2024-11-22T17:15:59,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/ccae1428ef6a490680707cb74caed610 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/ccae1428ef6a490680707cb74caed610 2024-11-22T17:15:59,279 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/ccae1428ef6a490680707cb74caed610, entries=150, sequenceid=83, filesize=11.7 K 2024-11-22T17:15:59,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/21efe3e3476944d6b3e44fd057028e1d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/21efe3e3476944d6b3e44fd057028e1d 2024-11-22T17:15:59,285 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/21efe3e3476944d6b3e44fd057028e1d, entries=150, sequenceid=83, filesize=11.7 K 2024-11-22T17:15:59,286 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2477a80617ad0e4131ef1992c2af963e in 789ms, sequenceid=83, compaction requested=false 2024-11-22T17:15:59,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:15:59,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:59,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-22T17:15:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-22T17:15:59,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-22T17:15:59,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5630 sec 2024-11-22T17:15:59,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.5670 sec 2024-11-22T17:15:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:59,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T17:15:59,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:15:59,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:15:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:15:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:15:59,416 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/64c28e0a96ce4e24af031e0295d221c1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/64c28e0a96ce4e24af031e0295d221c1 2024-11-22T17:15:59,421 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/A of 2477a80617ad0e4131ef1992c2af963e into 64c28e0a96ce4e24af031e0295d221c1(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:59,421 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:15:59,421 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/A, priority=13, startTime=1732295758409; duration=1sec 2024-11-22T17:15:59,421 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:59,421 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:A 2024-11-22T17:15:59,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112201215bc84719406d953229e137b56678_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295759407/Put/seqid=0 2024-11-22T17:15:59,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742143_1319 (size=17034) 2024-11-22T17:15:59,443 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:59,451 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112201215bc84719406d953229e137b56678_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112201215bc84719406d953229e137b56678_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:59,453 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5ace3af83efd4af49410f4ec6b8e5fc1, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:59,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5ace3af83efd4af49410f4ec6b8e5fc1 is 175, key is test_row_0/A:col10/1732295759407/Put/seqid=0 2024-11-22T17:15:59,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742144_1320 (size=48139) 2024-11-22T17:15:59,475 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5ace3af83efd4af49410f4ec6b8e5fc1 2024-11-22T17:15:59,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295819456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295819469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295819470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/28a758c6a4f2456e9ea51042abdfeebf is 50, key is test_row_0/B:col10/1732295759407/Put/seqid=0 2024-11-22T17:15:59,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295819477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295819477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742145_1321 (size=12001) 2024-11-22T17:15:59,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/28a758c6a4f2456e9ea51042abdfeebf 2024-11-22T17:15:59,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/5e8f42d15b6b42808d7eaf4d9a0d26d9 is 50, key is test_row_0/C:col10/1732295759407/Put/seqid=0 2024-11-22T17:15:59,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742146_1322 (size=12001) 2024-11-22T17:15:59,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/5e8f42d15b6b42808d7eaf4d9a0d26d9 2024-11-22T17:15:59,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5ace3af83efd4af49410f4ec6b8e5fc1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5ace3af83efd4af49410f4ec6b8e5fc1 2024-11-22T17:15:59,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5ace3af83efd4af49410f4ec6b8e5fc1, entries=250, sequenceid=99, filesize=47.0 K 2024-11-22T17:15:59,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/28a758c6a4f2456e9ea51042abdfeebf as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/28a758c6a4f2456e9ea51042abdfeebf 2024-11-22T17:15:59,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/28a758c6a4f2456e9ea51042abdfeebf, entries=150, sequenceid=99, filesize=11.7 K 2024-11-22T17:15:59,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/5e8f42d15b6b42808d7eaf4d9a0d26d9 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5e8f42d15b6b42808d7eaf4d9a0d26d9 2024-11-22T17:15:59,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5e8f42d15b6b42808d7eaf4d9a0d26d9, entries=150, sequenceid=99, filesize=11.7 K 2024-11-22T17:15:59,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 2477a80617ad0e4131ef1992c2af963e in 175ms, sequenceid=99, compaction requested=true 2024-11-22T17:15:59,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:15:59,585 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:59,586 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:59,586 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/A is initiating minor compaction (all files) 2024-11-22T17:15:59,586 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/A in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:59,587 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/64c28e0a96ce4e24af031e0295d221c1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/1046d1bd1317455e8795efa4d7f60818, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5ace3af83efd4af49410f4ec6b8e5fc1] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=107.6 K 2024-11-22T17:15:59,587 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:59,587 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/64c28e0a96ce4e24af031e0295d221c1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/1046d1bd1317455e8795efa4d7f60818, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5ace3af83efd4af49410f4ec6b8e5fc1] 2024-11-22T17:15:59,587 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c28e0a96ce4e24af031e0295d221c1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732295757338 2024-11-22T17:15:59,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:15:59,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:59,587 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:59,587 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1046d1bd1317455e8795efa4d7f60818, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732295757558 2024-11-22T17:15:59,588 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ace3af83efd4af49410f4ec6b8e5fc1, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732295758749 2024-11-22T17:15:59,588 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:59,588 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/B is initiating minor compaction (all files) 2024-11-22T17:15:59,588 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/B in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:59,588 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8e3de9dd7d7540139c6342c11c32bc12, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/ccae1428ef6a490680707cb74caed610, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/28a758c6a4f2456e9ea51042abdfeebf] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=35.3 K 2024-11-22T17:15:59,589 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e3de9dd7d7540139c6342c11c32bc12, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732295757338 2024-11-22T17:15:59,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:15:59,589 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting ccae1428ef6a490680707cb74caed610, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732295757558 2024-11-22T17:15:59,590 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 28a758c6a4f2456e9ea51042abdfeebf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732295758749 2024-11-22T17:15:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:59,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:59,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:15:59,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:59,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=114.05 KB heapSize=340.13 KB 2024-11-22T17:15:59,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295759466, max timestamp=1732295759596Num uniques 859; 2024-11-22T17:15:59,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295759466, max timestamp=1732295759596Num uniques 859; 2024-11-22T17:15:59,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295759466, max timestamp=1732295759596Num uniques 859; 2024-11-22T17:15:59,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:59,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:59,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:15:59,601 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:59,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:15:59,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:59,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:15:59,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:59,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:15:59,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:59,610 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#B#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:59,611 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/b6e3b5e157ca4e799f73481a7616fa1e is 50, key is test_row_0/B:col10/1732295759407/Put/seqid=0 2024-11-22T17:15:59,617 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122fcec3b3d844d4e7ba2f9d3daf0b4ac5b_2477a80617ad0e4131ef1992c2af963e store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:59,619 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122fcec3b3d844d4e7ba2f9d3daf0b4ac5b_2477a80617ad0e4131ef1992c2af963e, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:59,619 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122fcec3b3d844d4e7ba2f9d3daf0b4ac5b_2477a80617ad0e4131ef1992c2af963e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:59,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411221beb189c180743e5b6359468d01ba68b_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295759596/Put/seqid=0 2024-11-22T17:15:59,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742148_1324 (size=4469) 2024-11-22T17:15:59,632 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#A#compaction#273 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:59,633 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/fa6c14cd26284343b00cb67f9164d9d9 is 175, key is test_row_0/A:col10/1732295759407/Put/seqid=0 2024-11-22T17:15:59,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742147_1323 (size=12207) 2024-11-22T17:15:59,643 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/b6e3b5e157ca4e799f73481a7616fa1e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b6e3b5e157ca4e799f73481a7616fa1e 2024-11-22T17:15:59,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742149_1325 (size=17034) 2024-11-22T17:15:59,646 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:59,648 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/B of 2477a80617ad0e4131ef1992c2af963e into b6e3b5e157ca4e799f73481a7616fa1e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:59,648 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:15:59,648 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/B, priority=13, startTime=1732295759587; duration=0sec 2024-11-22T17:15:59,648 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:15:59,648 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:B 2024-11-22T17:15:59,648 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:15:59,649 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:15:59,649 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/C is initiating minor compaction (all files) 2024-11-22T17:15:59,649 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/C in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:15:59,650 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/157e182ebcda43df97e80c5c814e06ed, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/21efe3e3476944d6b3e44fd057028e1d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5e8f42d15b6b42808d7eaf4d9a0d26d9] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=35.3 K 2024-11-22T17:15:59,650 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 157e182ebcda43df97e80c5c814e06ed, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732295757338 2024-11-22T17:15:59,650 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411221beb189c180743e5b6359468d01ba68b_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411221beb189c180743e5b6359468d01ba68b_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:59,650 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 21efe3e3476944d6b3e44fd057028e1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732295757558 2024-11-22T17:15:59,651 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/09a4f51f6523496682b556226a683727, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:59,651 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e8f42d15b6b42808d7eaf4d9a0d26d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732295758749 2024-11-22T17:15:59,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/09a4f51f6523496682b556226a683727 is 175, key is test_row_0/A:col10/1732295759596/Put/seqid=0 2024-11-22T17:15:59,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742150_1326 (size=31161) 2024-11-22T17:15:59,670 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/fa6c14cd26284343b00cb67f9164d9d9 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/fa6c14cd26284343b00cb67f9164d9d9 2024-11-22T17:15:59,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742151_1327 (size=48139) 2024-11-22T17:15:59,671 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/09a4f51f6523496682b556226a683727 2024-11-22T17:15:59,675 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/A of 2477a80617ad0e4131ef1992c2af963e into fa6c14cd26284343b00cb67f9164d9d9(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:59,675 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:15:59,675 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/A, priority=13, startTime=1732295759585; duration=0sec 2024-11-22T17:15:59,675 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:59,675 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:A 2024-11-22T17:15:59,682 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#C#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:15:59,683 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/d2f196e758e9449c9174cab7324ff432 is 50, key is test_row_0/C:col10/1732295759407/Put/seqid=0 2024-11-22T17:15:59,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295759656, max timestamp=1732295759682Num uniques 859; 2024-11-22T17:15:59,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295759656, max timestamp=1732295759682Num uniques 859; 2024-11-22T17:15:59,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:15:59,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:15:59,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295759656, max timestamp=1732295759682Num uniques 859; 2024-11-22T17:15:59,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:15:59,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/0c507286ccbd4f0c828d7ac183e72dc4 is 50, key is test_row_0/B:col10/1732295759596/Put/seqid=0 2024-11-22T17:15:59,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742152_1328 (size=12207) 2024-11-22T17:15:59,704 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/d2f196e758e9449c9174cab7324ff432 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/d2f196e758e9449c9174cab7324ff432 2024-11-22T17:15:59,714 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/C of 2477a80617ad0e4131ef1992c2af963e into d2f196e758e9449c9174cab7324ff432(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:15:59,714 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:15:59,714 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/C, priority=13, startTime=1732295759598; duration=0sec 2024-11-22T17:15:59,714 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:15:59,714 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:C 2024-11-22T17:15:59,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742153_1329 (size=12001) 2024-11-22T17:15:59,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/0c507286ccbd4f0c828d7ac183e72dc4 2024-11-22T17:15:59,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/72ef0019aae64ecfbd2899ea797cbaca is 50, key is test_row_0/C:col10/1732295759596/Put/seqid=0 2024-11-22T17:15:59,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732295759735, max timestamp=1732295759755Num uniques 1018; 2024-11-22T17:15:59,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-22T17:15:59,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732295759735, max timestamp=1732295759755Num uniques 1018; 2024-11-22T17:15:59,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-22T17:15:59,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732295759735, max timestamp=1732295759755Num uniques 1018; 2024-11-22T17:15:59,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-22T17:15:59,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742154_1330 (size=12001) 2024-11-22T17:15:59,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/72ef0019aae64ecfbd2899ea797cbaca 2024-11-22T17:15:59,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/09a4f51f6523496682b556226a683727 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/09a4f51f6523496682b556226a683727 2024-11-22T17:15:59,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/09a4f51f6523496682b556226a683727, entries=250, sequenceid=121, filesize=47.0 K 2024-11-22T17:15:59,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/0c507286ccbd4f0c828d7ac183e72dc4 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/0c507286ccbd4f0c828d7ac183e72dc4 2024-11-22T17:15:59,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/0c507286ccbd4f0c828d7ac183e72dc4, entries=150, sequenceid=121, filesize=11.7 K 2024-11-22T17:15:59,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/72ef0019aae64ecfbd2899ea797cbaca as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/72ef0019aae64ecfbd2899ea797cbaca 2024-11-22T17:15:59,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/72ef0019aae64ecfbd2899ea797cbaca, entries=150, sequenceid=121, filesize=11.7 K 2024-11-22T17:15:59,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=107.34 KB/109920 for 2477a80617ad0e4131ef1992c2af963e in 201ms, sequenceid=121, compaction requested=false 2024-11-22T17:15:59,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:15:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:59,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=114.05 KB heapSize=319.69 KB 2024-11-22T17:15:59,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:15:59,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:59,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:15:59,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:59,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:15:59,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:15:59,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112279ab66ea535b44a9864dd3cb7a2f112a_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295759792/Put/seqid=0 2024-11-22T17:15:59,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742155_1331 (size=22268) 2024-11-22T17:15:59,818 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:15:59,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112279ab66ea535b44a9864dd3cb7a2f112a_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112279ab66ea535b44a9864dd3cb7a2f112a_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:15:59,824 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/72b07bdf84b1422da1f50ea5d07279d9, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:15:59,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/72b07bdf84b1422da1f50ea5d07279d9 is 175, key is test_row_0/A:col10/1732295759792/Put/seqid=0 2024-11-22T17:15:59,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295819826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742156_1332 (size=65673) 2024-11-22T17:15:59,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295819828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295819830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295819832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295819834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-22T17:15:59,871 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:15:59,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:15:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-22T17:15:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T17:15:59,873 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:15:59,874 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:15:59,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:15:59,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295819941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295819951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295819951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295819952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:15:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295819961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:15:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:15:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:15:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T17:16:00,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-22T17:16:00,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:00,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295820161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:00,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295820171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:00,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295820171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295820171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295820171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:00,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-22T17:16:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T17:16:00,242 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/72b07bdf84b1422da1f50ea5d07279d9 2024-11-22T17:16:00,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/ec15b3247f12443e99e1676d0e2e651b is 50, key is test_row_0/B:col10/1732295759792/Put/seqid=0 2024-11-22T17:16:00,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742157_1333 (size=12151) 2024-11-22T17:16:00,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-22T17:16:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295820471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:00,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:00,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-22T17:16:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295820482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:00,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295820482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:00,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295820482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295820482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T17:16:00,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-22T17:16:00,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:00,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/ec15b3247f12443e99e1676d0e2e651b 2024-11-22T17:16:00,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/5f445b2983e54ea1b3f0b0d157f652b5 is 50, key is test_row_0/C:col10/1732295759792/Put/seqid=0 2024-11-22T17:16:00,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742158_1334 (size=12151) 2024-11-22T17:16:00,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-22T17:16:00,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:00,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-22T17:16:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:00,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:00,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295820981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:00,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295820991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295820991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T17:16:01,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295821001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:01,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295821001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:01,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/5f445b2983e54ea1b3f0b0d157f652b5 2024-11-22T17:16:01,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/72b07bdf84b1422da1f50ea5d07279d9 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/72b07bdf84b1422da1f50ea5d07279d9 2024-11-22T17:16:01,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/72b07bdf84b1422da1f50ea5d07279d9, entries=350, sequenceid=174, filesize=64.1 K 2024-11-22T17:16:01,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/ec15b3247f12443e99e1676d0e2e651b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/ec15b3247f12443e99e1676d0e2e651b 2024-11-22T17:16:01,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/ec15b3247f12443e99e1676d0e2e651b, entries=150, sequenceid=174, filesize=11.9 K 2024-11-22T17:16:01,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/5f445b2983e54ea1b3f0b0d157f652b5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5f445b2983e54ea1b3f0b0d157f652b5 2024-11-22T17:16:01,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5f445b2983e54ea1b3f0b0d157f652b5, entries=150, sequenceid=174, filesize=11.9 K 2024-11-22T17:16:01,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~319.64 KB/327312, currentSize=67.09 KB/68700 for 2477a80617ad0e4131ef1992c2af963e in 1292ms, sequenceid=174, compaction requested=true 2024-11-22T17:16:01,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:01,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:16:01,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:01,093 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:01,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:16:01,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:01,093 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:01,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:16:01,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:01,094 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144973 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:01,094 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/A is initiating minor compaction (all files) 2024-11-22T17:16:01,094 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/A in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:01,094 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:01,094 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/B is initiating minor compaction (all files) 2024-11-22T17:16:01,094 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/fa6c14cd26284343b00cb67f9164d9d9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/09a4f51f6523496682b556226a683727, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/72b07bdf84b1422da1f50ea5d07279d9] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=141.6 K 2024-11-22T17:16:01,094 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/B in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:01,094 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:01,095 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/fa6c14cd26284343b00cb67f9164d9d9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/09a4f51f6523496682b556226a683727, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/72b07bdf84b1422da1f50ea5d07279d9] 2024-11-22T17:16:01,095 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b6e3b5e157ca4e799f73481a7616fa1e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/0c507286ccbd4f0c828d7ac183e72dc4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/ec15b3247f12443e99e1676d0e2e651b] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=35.5 K 2024-11-22T17:16:01,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-22T17:16:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:01,095 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa6c14cd26284343b00cb67f9164d9d9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732295758749 2024-11-22T17:16:01,095 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T17:16:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:16:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:16:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:01,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:16:01,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:01,096 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting b6e3b5e157ca4e799f73481a7616fa1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732295758749 2024-11-22T17:16:01,098 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09a4f51f6523496682b556226a683727, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732295759466 2024-11-22T17:16:01,098 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c507286ccbd4f0c828d7ac183e72dc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732295759468 2024-11-22T17:16:01,098 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72b07bdf84b1422da1f50ea5d07279d9, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732295759773 2024-11-22T17:16:01,099 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting ec15b3247f12443e99e1676d0e2e651b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732295759782 2024-11-22T17:16:01,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122fa6b3c2bc4e0413ba49d24b06154f2b0_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295759825/Put/seqid=0 2024-11-22T17:16:01,113 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:01,114 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#B#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:01,115 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/9c499625988a41cebf23ac58fb95df24 is 50, key is test_row_0/B:col10/1732295759792/Put/seqid=0 2024-11-22T17:16:01,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742159_1335 (size=12304) 2024-11-22T17:16:01,129 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122f7ce1924ab514a1fb07aee6d40a2d233_2477a80617ad0e4131ef1992c2af963e store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:01,131 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122f7ce1924ab514a1fb07aee6d40a2d233_2477a80617ad0e4131ef1992c2af963e, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:01,131 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f7ce1924ab514a1fb07aee6d40a2d233_2477a80617ad0e4131ef1992c2af963e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:01,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742160_1336 (size=12459) 2024-11-22T17:16:01,147 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/9c499625988a41cebf23ac58fb95df24 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/9c499625988a41cebf23ac58fb95df24 2024-11-22T17:16:01,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742161_1337 (size=4469) 2024-11-22T17:16:01,150 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#A#compaction#284 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:01,151 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/4dede57c5d52498690aeccb2dca8ee34 is 175, key is test_row_0/A:col10/1732295759792/Put/seqid=0 2024-11-22T17:16:01,158 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/B of 2477a80617ad0e4131ef1992c2af963e into 9c499625988a41cebf23ac58fb95df24(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:01,158 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:01,158 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/B, priority=13, startTime=1732295761093; duration=0sec 2024-11-22T17:16:01,159 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:01,159 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:B 2024-11-22T17:16:01,159 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:01,160 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:01,160 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/C is initiating minor compaction (all files) 2024-11-22T17:16:01,160 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/C in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:01,161 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/d2f196e758e9449c9174cab7324ff432, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/72ef0019aae64ecfbd2899ea797cbaca, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5f445b2983e54ea1b3f0b0d157f652b5] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=35.5 K 2024-11-22T17:16:01,161 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting d2f196e758e9449c9174cab7324ff432, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732295758749 2024-11-22T17:16:01,161 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 72ef0019aae64ecfbd2899ea797cbaca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732295759468 2024-11-22T17:16:01,162 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f445b2983e54ea1b3f0b0d157f652b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732295759782 2024-11-22T17:16:01,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742162_1338 (size=31413) 2024-11-22T17:16:01,179 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/4dede57c5d52498690aeccb2dca8ee34 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/4dede57c5d52498690aeccb2dca8ee34 2024-11-22T17:16:01,180 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:01,180 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/0138b0ed639b498883ef9b235e083fed is 50, key is test_row_0/C:col10/1732295759792/Put/seqid=0 2024-11-22T17:16:01,186 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/A of 2477a80617ad0e4131ef1992c2af963e into 4dede57c5d52498690aeccb2dca8ee34(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:01,186 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:01,186 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/A, priority=13, startTime=1732295761093; duration=0sec 2024-11-22T17:16:01,186 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:01,186 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:A 2024-11-22T17:16:01,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742163_1339 (size=12459) 2024-11-22T17:16:01,202 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/0138b0ed639b498883ef9b235e083fed as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/0138b0ed639b498883ef9b235e083fed 2024-11-22T17:16:01,206 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/C of 2477a80617ad0e4131ef1992c2af963e into 0138b0ed639b498883ef9b235e083fed(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:01,206 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:01,206 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/C, priority=13, startTime=1732295761093; duration=0sec 2024-11-22T17:16:01,206 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:01,206 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:C 2024-11-22T17:16:01,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:01,535 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122fa6b3c2bc4e0413ba49d24b06154f2b0_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fa6b3c2bc4e0413ba49d24b06154f2b0_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:01,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/f9056842f72046f899774e6e880e3287, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:01,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/f9056842f72046f899774e6e880e3287 is 175, key is test_row_0/A:col10/1732295759825/Put/seqid=0 2024-11-22T17:16:01,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742164_1340 (size=31105) 2024-11-22T17:16:01,958 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/f9056842f72046f899774e6e880e3287 2024-11-22T17:16:01,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/a1219c799e7641118802fe0925039859 is 50, key is test_row_0/B:col10/1732295759825/Put/seqid=0 2024-11-22T17:16:02,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742165_1341 (size=12151) 2024-11-22T17:16:02,012 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/a1219c799e7641118802fe0925039859 2024-11-22T17:16:02,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T17:16:02,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/e84727d8d4714616944079f476b09080 is 50, key is test_row_0/C:col10/1732295759825/Put/seqid=0 2024-11-22T17:16:02,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742166_1342 (size=12151) 2024-11-22T17:16:02,065 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/e84727d8d4714616944079f476b09080 2024-11-22T17:16:02,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/f9056842f72046f899774e6e880e3287 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/f9056842f72046f899774e6e880e3287 2024-11-22T17:16:02,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295822053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295822054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,075 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/f9056842f72046f899774e6e880e3287, entries=150, sequenceid=187, filesize=30.4 K 2024-11-22T17:16:02,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/a1219c799e7641118802fe0925039859 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/a1219c799e7641118802fe0925039859 2024-11-22T17:16:02,080 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/a1219c799e7641118802fe0925039859, entries=150, sequenceid=187, filesize=11.9 K 2024-11-22T17:16:02,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295822066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/e84727d8d4714616944079f476b09080 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/e84727d8d4714616944079f476b09080 2024-11-22T17:16:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295822068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295822069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,089 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/e84727d8d4714616944079f476b09080, entries=150, sequenceid=187, filesize=11.9 K 2024-11-22T17:16:02,090 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2477a80617ad0e4131ef1992c2af963e in 995ms, sequenceid=187, compaction requested=false 2024-11-22T17:16:02,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:02,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:02,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-22T17:16:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-22T17:16:02,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-22T17:16:02,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2180 sec 2024-11-22T17:16:02,098 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.2240 sec 2024-11-22T17:16:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:02,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-22T17:16:02,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:16:02,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:16:02,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:16:02,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:16:02,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:16:02,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:16:02,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a49df1881765466aa3334fa37ff3ad90_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295762034/Put/seqid=0 2024-11-22T17:16:02,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295822217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295822218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295822220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295822230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295822230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742167_1343 (size=17284) 2024-11-22T17:16:02,266 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:02,270 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a49df1881765466aa3334fa37ff3ad90_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a49df1881765466aa3334fa37ff3ad90_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:02,272 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/bbe21a32b4424962844c5832efc9a850, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:02,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/bbe21a32b4424962844c5832efc9a850 is 175, key is test_row_0/A:col10/1732295762034/Put/seqid=0 2024-11-22T17:16:02,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742168_1344 (size=48389) 2024-11-22T17:16:02,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295822341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295822351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295822351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295822351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295822351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295822551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295822561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295822561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295822571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295822572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,708 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/bbe21a32b4424962844c5832efc9a850 2024-11-22T17:16:02,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/8456d24ef7624072b72861f730a98aaf is 50, key is test_row_0/B:col10/1732295762034/Put/seqid=0 2024-11-22T17:16:02,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742169_1345 (size=12151) 2024-11-22T17:16:02,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295822861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295822871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295822871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295822891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295822891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:02,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/8456d24ef7624072b72861f730a98aaf 2024-11-22T17:16:03,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/a16776b985e04939a4efa21cf1281254 is 50, key is test_row_0/C:col10/1732295762034/Put/seqid=0 2024-11-22T17:16:03,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742170_1346 (size=12151) 2024-11-22T17:16:03,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/a16776b985e04939a4efa21cf1281254 2024-11-22T17:16:03,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/bbe21a32b4424962844c5832efc9a850 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/bbe21a32b4424962844c5832efc9a850 2024-11-22T17:16:03,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/bbe21a32b4424962844c5832efc9a850, entries=250, sequenceid=211, filesize=47.3 K 2024-11-22T17:16:03,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/8456d24ef7624072b72861f730a98aaf as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8456d24ef7624072b72861f730a98aaf 2024-11-22T17:16:03,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8456d24ef7624072b72861f730a98aaf, entries=150, sequenceid=211, filesize=11.9 K 2024-11-22T17:16:03,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/a16776b985e04939a4efa21cf1281254 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/a16776b985e04939a4efa21cf1281254 2024-11-22T17:16:03,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/a16776b985e04939a4efa21cf1281254, entries=150, sequenceid=211, filesize=11.9 K 2024-11-22T17:16:03,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 2477a80617ad0e4131ef1992c2af963e in 994ms, sequenceid=211, compaction requested=true 2024-11-22T17:16:03,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:03,189 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:03,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:16:03,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:03,190 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:03,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:16:03,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:03,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:16:03,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:03,191 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:03,191 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/A is initiating minor compaction (all files) 2024-11-22T17:16:03,191 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/A in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:03,191 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/4dede57c5d52498690aeccb2dca8ee34, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/f9056842f72046f899774e6e880e3287, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/bbe21a32b4424962844c5832efc9a850] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=108.3 K 2024-11-22T17:16:03,191 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:03,191 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:03,191 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/B is initiating minor compaction (all files) 2024-11-22T17:16:03,191 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/4dede57c5d52498690aeccb2dca8ee34, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/f9056842f72046f899774e6e880e3287, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/bbe21a32b4424962844c5832efc9a850] 2024-11-22T17:16:03,191 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/B in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:03,191 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/9c499625988a41cebf23ac58fb95df24, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/a1219c799e7641118802fe0925039859, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8456d24ef7624072b72861f730a98aaf] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=35.9 K 2024-11-22T17:16:03,192 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dede57c5d52498690aeccb2dca8ee34, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732295759782 2024-11-22T17:16:03,192 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c499625988a41cebf23ac58fb95df24, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732295759782 2024-11-22T17:16:03,192 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9056842f72046f899774e6e880e3287, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732295759825 2024-11-22T17:16:03,192 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a1219c799e7641118802fe0925039859, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732295759825 2024-11-22T17:16:03,193 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbe21a32b4424962844c5832efc9a850, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732295762034 2024-11-22T17:16:03,193 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 8456d24ef7624072b72861f730a98aaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732295762034 2024-11-22T17:16:03,204 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#B#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:03,204 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/b3db6e278ae54627871aec2f56c85b88 is 50, key is test_row_0/B:col10/1732295762034/Put/seqid=0 2024-11-22T17:16:03,206 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:03,220 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122f55c7bfcb5af42079cb0cd020a550058_2477a80617ad0e4131ef1992c2af963e store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:03,222 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122f55c7bfcb5af42079cb0cd020a550058_2477a80617ad0e4131ef1992c2af963e, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:03,223 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f55c7bfcb5af42079cb0cd020a550058_2477a80617ad0e4131ef1992c2af963e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:03,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742171_1347 (size=12561) 2024-11-22T17:16:03,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742172_1348 (size=4469) 2024-11-22T17:16:03,242 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#A#compaction#292 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:03,243 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/88f85d47edd8438488c4b83c0e4185ef is 175, key is test_row_0/A:col10/1732295762034/Put/seqid=0 2024-11-22T17:16:03,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742173_1349 (size=31515) 2024-11-22T17:16:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:03,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T17:16:03,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:16:03,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:03,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:16:03,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:03,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:16:03,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:03,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e4022fe039eb42eb919016bc1b699501_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295763383/Put/seqid=0 2024-11-22T17:16:03,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742174_1350 (size=14794) 2024-11-22T17:16:03,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295823426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295823428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295823428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295823429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295823436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295823551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295823551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295823551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295823551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295823551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,635 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/b3db6e278ae54627871aec2f56c85b88 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b3db6e278ae54627871aec2f56c85b88 2024-11-22T17:16:03,639 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/B of 2477a80617ad0e4131ef1992c2af963e into b3db6e278ae54627871aec2f56c85b88(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:03,639 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:03,639 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/B, priority=13, startTime=1732295763190; duration=0sec 2024-11-22T17:16:03,639 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:03,639 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:B 2024-11-22T17:16:03,639 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:03,640 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:03,640 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/C is initiating minor compaction (all files) 2024-11-22T17:16:03,640 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/C in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:03,640 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/0138b0ed639b498883ef9b235e083fed, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/e84727d8d4714616944079f476b09080, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/a16776b985e04939a4efa21cf1281254] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=35.9 K 2024-11-22T17:16:03,640 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 0138b0ed639b498883ef9b235e083fed, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732295759782 2024-11-22T17:16:03,641 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting e84727d8d4714616944079f476b09080, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732295759825 2024-11-22T17:16:03,641 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a16776b985e04939a4efa21cf1281254, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732295762034 2024-11-22T17:16:03,646 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:03,647 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/4427d39e134b44d6bdafa53e6437d91f is 50, key is test_row_0/C:col10/1732295762034/Put/seqid=0 2024-11-22T17:16:03,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742175_1351 (size=12561) 2024-11-22T17:16:03,667 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/88f85d47edd8438488c4b83c0e4185ef as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/88f85d47edd8438488c4b83c0e4185ef 2024-11-22T17:16:03,670 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/A of 2477a80617ad0e4131ef1992c2af963e into 88f85d47edd8438488c4b83c0e4185ef(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:03,670 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:03,670 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/A, priority=13, startTime=1732295763189; duration=0sec 2024-11-22T17:16:03,670 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:03,670 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:A 2024-11-22T17:16:03,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295823771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295823771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295823771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295823771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295823772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:03,805 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:03,808 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e4022fe039eb42eb919016bc1b699501_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e4022fe039eb42eb919016bc1b699501_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:03,809 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/32e0e1f56e7a4df5a278bd97b2caf765, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:03,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/32e0e1f56e7a4df5a278bd97b2caf765 is 175, key is test_row_0/A:col10/1732295763383/Put/seqid=0 2024-11-22T17:16:03,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742176_1352 (size=39749) 2024-11-22T17:16:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T17:16:04,041 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:04,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:16:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-22T17:16:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T17:16:04,044 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:16:04,045 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:16:04,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:16:04,057 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/4427d39e134b44d6bdafa53e6437d91f as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/4427d39e134b44d6bdafa53e6437d91f 2024-11-22T17:16:04,068 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/C of 2477a80617ad0e4131ef1992c2af963e into 4427d39e134b44d6bdafa53e6437d91f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:04,068 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:04,068 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/C, priority=13, startTime=1732295763190; duration=0sec 2024-11-22T17:16:04,068 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:04,068 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:C 2024-11-22T17:16:04,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295824081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:04,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295824082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:04,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295824092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:04,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295824092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:04,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295824097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T17:16:04,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T17:16:04,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:04,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:04,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:04,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:04,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:04,219 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/32e0e1f56e7a4df5a278bd97b2caf765 2024-11-22T17:16:04,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/1f86388675974107a2de37a594cc709a is 50, key is test_row_0/B:col10/1732295763383/Put/seqid=0 2024-11-22T17:16:04,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742177_1353 (size=12151) 2024-11-22T17:16:04,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/1f86388675974107a2de37a594cc709a 2024-11-22T17:16:04,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/c36f34ef5db3492ebc16d5d0fd97a2ae is 50, key is test_row_0/C:col10/1732295763383/Put/seqid=0 2024-11-22T17:16:04,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742178_1354 (size=12151) 2024-11-22T17:16:04,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T17:16:04,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:04,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:04,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:04,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:04,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T17:16:04,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T17:16:04,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:04,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:04,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:04,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:04,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:04,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295824602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:04,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295824602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295824602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:04,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295824611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:04,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:04,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295824621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:04,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/c36f34ef5db3492ebc16d5d0fd97a2ae 2024-11-22T17:16:04,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/32e0e1f56e7a4df5a278bd97b2caf765 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/32e0e1f56e7a4df5a278bd97b2caf765 2024-11-22T17:16:04,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/32e0e1f56e7a4df5a278bd97b2caf765, entries=200, sequenceid=223, filesize=38.8 K 2024-11-22T17:16:04,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/1f86388675974107a2de37a594cc709a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/1f86388675974107a2de37a594cc709a 2024-11-22T17:16:04,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/1f86388675974107a2de37a594cc709a, entries=150, sequenceid=223, filesize=11.9 K 2024-11-22T17:16:04,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/c36f34ef5db3492ebc16d5d0fd97a2ae as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/c36f34ef5db3492ebc16d5d0fd97a2ae 2024-11-22T17:16:04,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T17:16:04,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:04,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:04,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:04,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:04,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/c36f34ef5db3492ebc16d5d0fd97a2ae, entries=150, sequenceid=223, filesize=11.9 K 2024-11-22T17:16:04,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:04,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2477a80617ad0e4131ef1992c2af963e in 1273ms, sequenceid=223, compaction requested=false 2024-11-22T17:16:04,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T17:16:04,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T17:16:04,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:04,809 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T17:16:04,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:16:04,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:04,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:16:04,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:04,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:16:04,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:04,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222e3933f68a1a4e63859c85ff8c33b6e1_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295763435/Put/seqid=0 2024-11-22T17:16:04,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742179_1355 (size=12304) 2024-11-22T17:16:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T17:16:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:05,222 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222e3933f68a1a4e63859c85ff8c33b6e1_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222e3933f68a1a4e63859c85ff8c33b6e1_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:05,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5fdab12892bc4f4ba74509e917d0e262, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:05,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5fdab12892bc4f4ba74509e917d0e262 is 175, key is test_row_0/A:col10/1732295763435/Put/seqid=0 2024-11-22T17:16:05,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742180_1356 (size=31105) 2024-11-22T17:16:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:05,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:05,628 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5fdab12892bc4f4ba74509e917d0e262 2024-11-22T17:16:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/970300e115db4d91b9a0a7b205fc925a is 50, key is test_row_0/B:col10/1732295763435/Put/seqid=0 2024-11-22T17:16:05,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742181_1357 (size=12151) 2024-11-22T17:16:05,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295825636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295825644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295825644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295825645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295825645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295825751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295825751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295825751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295825761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295825771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295825961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295825961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295825961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295825971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295825981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,042 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/970300e115db4d91b9a0a7b205fc925a 2024-11-22T17:16:06,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/c0fb6fce110e4c839826fbd9321dc4be is 50, key is test_row_0/C:col10/1732295763435/Put/seqid=0 2024-11-22T17:16:06,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742182_1358 (size=12151) 2024-11-22T17:16:06,103 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/c0fb6fce110e4c839826fbd9321dc4be 2024-11-22T17:16:06,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5fdab12892bc4f4ba74509e917d0e262 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5fdab12892bc4f4ba74509e917d0e262 2024-11-22T17:16:06,111 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5fdab12892bc4f4ba74509e917d0e262, entries=150, sequenceid=246, filesize=30.4 K 2024-11-22T17:16:06,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/970300e115db4d91b9a0a7b205fc925a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/970300e115db4d91b9a0a7b205fc925a 2024-11-22T17:16:06,120 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/970300e115db4d91b9a0a7b205fc925a, entries=150, sequenceid=246, filesize=11.9 K 2024-11-22T17:16:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/c0fb6fce110e4c839826fbd9321dc4be as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/c0fb6fce110e4c839826fbd9321dc4be 2024-11-22T17:16:06,128 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/c0fb6fce110e4c839826fbd9321dc4be, entries=150, sequenceid=246, filesize=11.9 K 2024-11-22T17:16:06,130 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2477a80617ad0e4131ef1992c2af963e in 1321ms, sequenceid=246, compaction requested=true 2024-11-22T17:16:06,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:06,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-22T17:16:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-22T17:16:06,134 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-22T17:16:06,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0870 sec 2024-11-22T17:16:06,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.0940 sec 2024-11-22T17:16:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T17:16:06,192 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:06,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:16:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-22T17:16:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T17:16:06,196 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:16:06,196 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:16:06,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:16:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:06,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T17:16:06,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:16:06,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:06,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:16:06,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:06,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:16:06,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:06,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228b978d7da8e54371bd9458106d5282b9_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295766272/Put/seqid=0 2024-11-22T17:16:06,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742183_1359 (size=14894) 2024-11-22T17:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T17:16:06,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295826307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295826308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295826310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295826315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295826316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T17:16:06,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:06,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:06,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:06,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295826431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295826431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295826431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295826433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295826433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T17:16:06,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:06,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:06,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:06,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T17:16:06,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295826651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295826651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295826652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295826652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T17:16:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:06,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295826652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:06,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,685 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:06,691 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228b978d7da8e54371bd9458106d5282b9_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228b978d7da8e54371bd9458106d5282b9_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:06,692 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/3202f3c3877a400f88643e86d575146f, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:06,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/3202f3c3877a400f88643e86d575146f is 175, key is test_row_0/A:col10/1732295766272/Put/seqid=0 2024-11-22T17:16:06,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742184_1360 (size=39849) 2024-11-22T17:16:06,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T17:16:06,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:06,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:06,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:06,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T17:16:06,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295826961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295826961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T17:16:06,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:06,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:06,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:06,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:06,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295826971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295826971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295826972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:07,115 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/3202f3c3877a400f88643e86d575146f 2024-11-22T17:16:07,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T17:16:07,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:07,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/c18b2f0e35b44596a1cac2bbb9558a32 is 50, key is test_row_0/B:col10/1732295766272/Put/seqid=0 2024-11-22T17:16:07,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742185_1361 (size=12251) 2024-11-22T17:16:07,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T17:16:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T17:16:07,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T17:16:07,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:07,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295827471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:07,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295827471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:07,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295827482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:07,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295827482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:07,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295827482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:07,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/c18b2f0e35b44596a1cac2bbb9558a32 2024-11-22T17:16:07,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/549eada901a24d40a59734ff3f77c2c9 is 50, key is test_row_0/C:col10/1732295766272/Put/seqid=0 2024-11-22T17:16:07,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742186_1362 (size=12251) 2024-11-22T17:16:07,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T17:16:07,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:07,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T17:16:07,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:07,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T17:16:07,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:07,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:07,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/549eada901a24d40a59734ff3f77c2c9 2024-11-22T17:16:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/3202f3c3877a400f88643e86d575146f as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/3202f3c3877a400f88643e86d575146f 2024-11-22T17:16:07,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/3202f3c3877a400f88643e86d575146f, entries=200, sequenceid=259, filesize=38.9 K 2024-11-22T17:16:07,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/c18b2f0e35b44596a1cac2bbb9558a32 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/c18b2f0e35b44596a1cac2bbb9558a32 2024-11-22T17:16:07,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/c18b2f0e35b44596a1cac2bbb9558a32, entries=150, sequenceid=259, filesize=12.0 K 2024-11-22T17:16:07,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/549eada901a24d40a59734ff3f77c2c9 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/549eada901a24d40a59734ff3f77c2c9 2024-11-22T17:16:07,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/549eada901a24d40a59734ff3f77c2c9, entries=150, sequenceid=259, filesize=12.0 K 2024-11-22T17:16:07,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2477a80617ad0e4131ef1992c2af963e in 1713ms, sequenceid=259, compaction requested=true 2024-11-22T17:16:07,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:07,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:16:07,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:07,987 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:16:07,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:16:07,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:07,987 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:16:07,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:16:07,988 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142218 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:16:07,988 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/A is initiating minor compaction (all files) 2024-11-22T17:16:07,988 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/A in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,988 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/88f85d47edd8438488c4b83c0e4185ef, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/32e0e1f56e7a4df5a278bd97b2caf765, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5fdab12892bc4f4ba74509e917d0e262, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/3202f3c3877a400f88643e86d575146f] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=138.9 K 2024-11-22T17:16:07,988 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,988 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/88f85d47edd8438488c4b83c0e4185ef, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/32e0e1f56e7a4df5a278bd97b2caf765, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5fdab12892bc4f4ba74509e917d0e262, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/3202f3c3877a400f88643e86d575146f] 2024-11-22T17:16:07,988 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49114 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:16:07,988 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/B is initiating minor compaction (all files) 2024-11-22T17:16:07,988 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88f85d47edd8438488c4b83c0e4185ef, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732295762034 2024-11-22T17:16:07,988 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/B in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:07,989 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b3db6e278ae54627871aec2f56c85b88, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/1f86388675974107a2de37a594cc709a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/970300e115db4d91b9a0a7b205fc925a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/c18b2f0e35b44596a1cac2bbb9558a32] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=48.0 K 2024-11-22T17:16:07,989 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32e0e1f56e7a4df5a278bd97b2caf765, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732295762212 2024-11-22T17:16:07,989 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting b3db6e278ae54627871aec2f56c85b88, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732295762034 2024-11-22T17:16:07,989 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fdab12892bc4f4ba74509e917d0e262, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732295763426 2024-11-22T17:16:07,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:07,989 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f86388675974107a2de37a594cc709a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732295762212 2024-11-22T17:16:07,989 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3202f3c3877a400f88643e86d575146f, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732295765626 2024-11-22T17:16:07,989 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 970300e115db4d91b9a0a7b205fc925a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732295763426 2024-11-22T17:16:07,990 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting c18b2f0e35b44596a1cac2bbb9558a32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732295765626 2024-11-22T17:16:07,999 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:08,000 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/678b20a268b54beda0238c6616a1289b is 50, key is test_row_0/B:col10/1732295766272/Put/seqid=0 2024-11-22T17:16:08,001 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:08,008 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411227e8a1818ea054be2b2d94c084af0dee3_2477a80617ad0e4131ef1992c2af963e store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:08,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742187_1363 (size=12797) 2024-11-22T17:16:08,011 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411227e8a1818ea054be2b2d94c084af0dee3_2477a80617ad0e4131ef1992c2af963e, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:08,011 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227e8a1818ea054be2b2d94c084af0dee3_2477a80617ad0e4131ef1992c2af963e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:08,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742188_1364 (size=4469) 2024-11-22T17:16:08,018 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/678b20a268b54beda0238c6616a1289b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/678b20a268b54beda0238c6616a1289b 2024-11-22T17:16:08,023 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/B of 2477a80617ad0e4131ef1992c2af963e into 678b20a268b54beda0238c6616a1289b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:08,024 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:08,024 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/B, priority=12, startTime=1732295767987; duration=0sec 2024-11-22T17:16:08,024 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:08,024 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:B 2024-11-22T17:16:08,024 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:16:08,025 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49114 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:16:08,025 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/C is initiating minor compaction (all files) 2024-11-22T17:16:08,025 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/C in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:08,025 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/4427d39e134b44d6bdafa53e6437d91f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/c36f34ef5db3492ebc16d5d0fd97a2ae, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/c0fb6fce110e4c839826fbd9321dc4be, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/549eada901a24d40a59734ff3f77c2c9] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=48.0 K 2024-11-22T17:16:08,025 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 4427d39e134b44d6bdafa53e6437d91f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732295762034 2024-11-22T17:16:08,026 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting c36f34ef5db3492ebc16d5d0fd97a2ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732295762212 2024-11-22T17:16:08,026 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting c0fb6fce110e4c839826fbd9321dc4be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732295763426 2024-11-22T17:16:08,026 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 549eada901a24d40a59734ff3f77c2c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732295765626 2024-11-22T17:16:08,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T17:16:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:08,034 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T17:16:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:16:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:16:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:16:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:08,035 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#C#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:08,036 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/8a5286532f8e479799f1bc2d6bfb9b55 is 50, key is test_row_0/C:col10/1732295766272/Put/seqid=0 2024-11-22T17:16:08,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742189_1365 (size=12797) 2024-11-22T17:16:08,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226b7916c308f64022a79b536d2d283962_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295766309/Put/seqid=0 2024-11-22T17:16:08,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742190_1366 (size=12454) 2024-11-22T17:16:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T17:16:08,415 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#A#compaction#304 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:08,416 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/11e176a36ede49699d1a5b6fc83154ba is 175, key is test_row_0/A:col10/1732295766272/Put/seqid=0 2024-11-22T17:16:08,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742191_1367 (size=31751) 2024-11-22T17:16:08,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:08,456 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/8a5286532f8e479799f1bc2d6bfb9b55 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/8a5286532f8e479799f1bc2d6bfb9b55 2024-11-22T17:16:08,458 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226b7916c308f64022a79b536d2d283962_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226b7916c308f64022a79b536d2d283962_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:08,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/75187badf3674602b33de3833fcf5cac, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:08,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/75187badf3674602b33de3833fcf5cac is 175, key is test_row_0/A:col10/1732295766309/Put/seqid=0 2024-11-22T17:16:08,461 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/C of 2477a80617ad0e4131ef1992c2af963e into 8a5286532f8e479799f1bc2d6bfb9b55(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:08,461 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:08,461 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/C, priority=12, startTime=1732295767987; duration=0sec 2024-11-22T17:16:08,461 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:08,461 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:C 2024-11-22T17:16:08,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742192_1368 (size=31255) 2024-11-22T17:16:08,468 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/75187badf3674602b33de3833fcf5cac 2024-11-22T17:16:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:08,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:08,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/b3b13a01c154452cb0d0cfe7689ec9e0 is 50, key is test_row_0/B:col10/1732295766309/Put/seqid=0 2024-11-22T17:16:08,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742193_1369 (size=12301) 2024-11-22T17:16:08,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295828510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295828511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295828513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295828513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295828514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295828621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295828631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295828631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295828632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295828632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,825 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/11e176a36ede49699d1a5b6fc83154ba as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/11e176a36ede49699d1a5b6fc83154ba 2024-11-22T17:16:08,830 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/A of 2477a80617ad0e4131ef1992c2af963e into 11e176a36ede49699d1a5b6fc83154ba(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:08,830 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:08,830 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/A, priority=12, startTime=1732295767986; duration=0sec 2024-11-22T17:16:08,830 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:08,830 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:A 2024-11-22T17:16:08,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295828841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295828852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295828852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295828852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295828852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:08,895 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/b3b13a01c154452cb0d0cfe7689ec9e0 2024-11-22T17:16:08,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/46e5522672094259852edf9bf5b51559 is 50, key is test_row_0/C:col10/1732295766309/Put/seqid=0 2024-11-22T17:16:08,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742194_1370 (size=12301) 2024-11-22T17:16:09,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295829161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295829171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295829172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295829172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295829172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,305 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/46e5522672094259852edf9bf5b51559 2024-11-22T17:16:09,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/75187badf3674602b33de3833fcf5cac as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/75187badf3674602b33de3833fcf5cac 2024-11-22T17:16:09,313 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/75187badf3674602b33de3833fcf5cac, entries=150, sequenceid=279, filesize=30.5 K 2024-11-22T17:16:09,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/b3b13a01c154452cb0d0cfe7689ec9e0 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b3b13a01c154452cb0d0cfe7689ec9e0 2024-11-22T17:16:09,317 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b3b13a01c154452cb0d0cfe7689ec9e0, entries=150, sequenceid=279, filesize=12.0 K 2024-11-22T17:16:09,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/46e5522672094259852edf9bf5b51559 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/46e5522672094259852edf9bf5b51559 2024-11-22T17:16:09,321 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/46e5522672094259852edf9bf5b51559, entries=150, sequenceid=279, filesize=12.0 K 2024-11-22T17:16:09,322 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2477a80617ad0e4131ef1992c2af963e in 1288ms, sequenceid=279, compaction requested=false 2024-11-22T17:16:09,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:09,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:09,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-22T17:16:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-22T17:16:09,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-22T17:16:09,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1260 sec 2024-11-22T17:16:09,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.1330 sec 2024-11-22T17:16:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:09,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T17:16:09,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:16:09,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:09,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:16:09,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:09,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:16:09,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:09,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122eae666db719946c7bcfde22a77ae1367_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295769683/Put/seqid=0 2024-11-22T17:16:09,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742195_1371 (size=14994) 2024-11-22T17:16:09,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295829717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295829718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295829718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295829719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295829724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295829831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295829842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295829842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295829842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295829842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295830041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295830052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295830052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295830052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295830052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,096 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:10,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122eae666db719946c7bcfde22a77ae1367_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122eae666db719946c7bcfde22a77ae1367_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:10,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/b98d22e7a31948348a199e29178f60fe, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:10,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/b98d22e7a31948348a199e29178f60fe is 175, key is test_row_0/A:col10/1732295769683/Put/seqid=0 2024-11-22T17:16:10,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742196_1372 (size=39949) 2024-11-22T17:16:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T17:16:10,361 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:10,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:16:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-22T17:16:10,363 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:16:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T17:16:10,364 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:16:10,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:16:10,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295830361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295830361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295830372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295830372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295830372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T17:16:10,504 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/b98d22e7a31948348a199e29178f60fe 2024-11-22T17:16:10,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/f76f35d9f4244caeb1f0ee2033322a51 is 50, key is test_row_0/B:col10/1732295769683/Put/seqid=0 2024-11-22T17:16:10,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T17:16:10,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:10,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:10,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:10,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:10,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:10,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742197_1373 (size=12301) 2024-11-22T17:16:10,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T17:16:10,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:10,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:10,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:10,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:10,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T17:16:10,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T17:16:10,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:10,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:10,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:10,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:10,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:10,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295830881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295830881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295830891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295830891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:10,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295830892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:10,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/f76f35d9f4244caeb1f0ee2033322a51 2024-11-22T17:16:10,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/d32aafa01a06479392c02abae2d57d5c is 50, key is test_row_0/C:col10/1732295769683/Put/seqid=0 2024-11-22T17:16:10,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742198_1374 (size=12301) 2024-11-22T17:16:10,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T17:16:10,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:10,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:10,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:10,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:10,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T17:16:11,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T17:16:11,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:11,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:11,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:11,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:11,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:11,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T17:16:11,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:11,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:11,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:11,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:11,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:11,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/d32aafa01a06479392c02abae2d57d5c 2024-11-22T17:16:11,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/b98d22e7a31948348a199e29178f60fe as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/b98d22e7a31948348a199e29178f60fe 2024-11-22T17:16:11,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/b98d22e7a31948348a199e29178f60fe, entries=200, sequenceid=295, filesize=39.0 K 2024-11-22T17:16:11,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/f76f35d9f4244caeb1f0ee2033322a51 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/f76f35d9f4244caeb1f0ee2033322a51 2024-11-22T17:16:11,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/f76f35d9f4244caeb1f0ee2033322a51, entries=150, sequenceid=295, filesize=12.0 K 2024-11-22T17:16:11,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/d32aafa01a06479392c02abae2d57d5c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/d32aafa01a06479392c02abae2d57d5c 2024-11-22T17:16:11,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/d32aafa01a06479392c02abae2d57d5c, entries=150, sequenceid=295, filesize=12.0 K 2024-11-22T17:16:11,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2477a80617ad0e4131ef1992c2af963e in 1669ms, sequenceid=295, compaction requested=true 2024-11-22T17:16:11,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:11,353 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:11,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:16:11,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:11,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:16:11,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:11,354 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:11,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:16:11,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:11,356 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:11,356 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/B is initiating minor compaction (all files) 2024-11-22T17:16:11,356 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/B in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:11,357 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/678b20a268b54beda0238c6616a1289b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b3b13a01c154452cb0d0cfe7689ec9e0, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/f76f35d9f4244caeb1f0ee2033322a51] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=36.5 K 2024-11-22T17:16:11,357 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102955 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:11,357 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/A is initiating minor compaction (all files) 2024-11-22T17:16:11,357 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/A in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:11,357 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 678b20a268b54beda0238c6616a1289b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732295765626 2024-11-22T17:16:11,357 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/11e176a36ede49699d1a5b6fc83154ba, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/75187badf3674602b33de3833fcf5cac, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/b98d22e7a31948348a199e29178f60fe] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=100.5 K 2024-11-22T17:16:11,357 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:11,357 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/11e176a36ede49699d1a5b6fc83154ba, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/75187badf3674602b33de3833fcf5cac, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/b98d22e7a31948348a199e29178f60fe] 2024-11-22T17:16:11,357 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting b3b13a01c154452cb0d0cfe7689ec9e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732295766297 2024-11-22T17:16:11,361 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11e176a36ede49699d1a5b6fc83154ba, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732295765626 2024-11-22T17:16:11,361 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting f76f35d9f4244caeb1f0ee2033322a51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732295768511 2024-11-22T17:16:11,361 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75187badf3674602b33de3833fcf5cac, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732295766297 2024-11-22T17:16:11,361 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting b98d22e7a31948348a199e29178f60fe, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732295768495 2024-11-22T17:16:11,370 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:11,373 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#B#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:11,374 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/99bc7d2a5d234aa7a80f939449583f6a is 50, key is test_row_0/B:col10/1732295769683/Put/seqid=0 2024-11-22T17:16:11,376 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122cdbd477fec894db89f4ceeb0728a4b46_2477a80617ad0e4131ef1992c2af963e store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:11,378 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122cdbd477fec894db89f4ceeb0728a4b46_2477a80617ad0e4131ef1992c2af963e, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:11,378 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122cdbd477fec894db89f4ceeb0728a4b46_2477a80617ad0e4131ef1992c2af963e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:11,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742199_1375 (size=12949) 2024-11-22T17:16:11,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742200_1376 (size=4469) 2024-11-22T17:16:11,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T17:16:11,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:11,432 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T17:16:11,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:16:11,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:11,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:16:11,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:11,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:16:11,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:11,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112220b62df0c7ef471b84fc97d489eab22e_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295769715/Put/seqid=0 2024-11-22T17:16:11,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742201_1377 (size=12454) 2024-11-22T17:16:11,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:11,446 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112220b62df0c7ef471b84fc97d489eab22e_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112220b62df0c7ef471b84fc97d489eab22e_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/195942a783f14e97bd60a6b479c1d02d, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:11,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/195942a783f14e97bd60a6b479c1d02d is 175, key is test_row_0/A:col10/1732295769715/Put/seqid=0 2024-11-22T17:16:11,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742202_1378 (size=31255) 2024-11-22T17:16:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T17:16:11,791 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#A#compaction#312 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:11,791 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/301a85fb85024c1ea1f8dbf4c8ffba65 is 175, key is test_row_0/A:col10/1732295769683/Put/seqid=0 2024-11-22T17:16:11,792 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/99bc7d2a5d234aa7a80f939449583f6a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/99bc7d2a5d234aa7a80f939449583f6a 2024-11-22T17:16:11,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742203_1379 (size=31903) 2024-11-22T17:16:11,799 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/B of 2477a80617ad0e4131ef1992c2af963e into 99bc7d2a5d234aa7a80f939449583f6a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:11,799 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:11,799 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/B, priority=13, startTime=1732295771354; duration=0sec 2024-11-22T17:16:11,799 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:11,799 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:B 2024-11-22T17:16:11,799 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:11,799 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/301a85fb85024c1ea1f8dbf4c8ffba65 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/301a85fb85024c1ea1f8dbf4c8ffba65 2024-11-22T17:16:11,804 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:11,804 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/C is initiating minor compaction (all files) 2024-11-22T17:16:11,804 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/C in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:11,804 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/8a5286532f8e479799f1bc2d6bfb9b55, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/46e5522672094259852edf9bf5b51559, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/d32aafa01a06479392c02abae2d57d5c] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=36.5 K 2024-11-22T17:16:11,805 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a5286532f8e479799f1bc2d6bfb9b55, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732295765626 2024-11-22T17:16:11,805 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e5522672094259852edf9bf5b51559, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732295766297 2024-11-22T17:16:11,806 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting d32aafa01a06479392c02abae2d57d5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732295768511 2024-11-22T17:16:11,808 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/A of 2477a80617ad0e4131ef1992c2af963e into 301a85fb85024c1ea1f8dbf4c8ffba65(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:11,808 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:11,808 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/A, priority=13, startTime=1732295771353; duration=0sec 2024-11-22T17:16:11,808 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:11,808 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:A 2024-11-22T17:16:11,813 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#C#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:11,814 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/513eabf0846f4d319bff35a39f6a74f4 is 50, key is test_row_0/C:col10/1732295769683/Put/seqid=0 2024-11-22T17:16:11,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742204_1380 (size=12949) 2024-11-22T17:16:11,851 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/195942a783f14e97bd60a6b479c1d02d 2024-11-22T17:16:11,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/c070da46e81f46a4ab1b8ecc5c123e14 is 50, key is test_row_0/B:col10/1732295769715/Put/seqid=0 2024-11-22T17:16:11,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742205_1381 (size=12301) 2024-11-22T17:16:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:11,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:11,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295831921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:11,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295831921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:11,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295831922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:11,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295831927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:11,932 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_0' in 2477a80617ad0e4131ef1992c2af963e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:11,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295831931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:12,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295832041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:12,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:12,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295832041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:12,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:12,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295832041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:12,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:12,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295832051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:12,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:12,225 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/513eabf0846f4d319bff35a39f6a74f4 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/513eabf0846f4d319bff35a39f6a74f4 2024-11-22T17:16:12,232 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/C of 2477a80617ad0e4131ef1992c2af963e into 513eabf0846f4d319bff35a39f6a74f4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:12,232 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:12,232 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/C, priority=13, startTime=1732295771354; duration=0sec 2024-11-22T17:16:12,232 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:12,232 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:C 2024-11-22T17:16:12,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295832253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:12,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295832253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:12,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295832254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295832261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:12,274 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/c070da46e81f46a4ab1b8ecc5c123e14 2024-11-22T17:16:12,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/5c7516f7fc8241cf898f19e95771c198 is 50, key is test_row_0/C:col10/1732295769715/Put/seqid=0 2024-11-22T17:16:12,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742206_1382 (size=12301) 2024-11-22T17:16:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T17:16:12,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295832571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295832571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295832571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:12,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295832571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:12,683 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/5c7516f7fc8241cf898f19e95771c198 2024-11-22T17:16:12,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/195942a783f14e97bd60a6b479c1d02d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/195942a783f14e97bd60a6b479c1d02d 2024-11-22T17:16:12,694 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/195942a783f14e97bd60a6b479c1d02d, entries=150, sequenceid=313, filesize=30.5 K 2024-11-22T17:16:12,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/c070da46e81f46a4ab1b8ecc5c123e14 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/c070da46e81f46a4ab1b8ecc5c123e14 2024-11-22T17:16:12,698 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/c070da46e81f46a4ab1b8ecc5c123e14, entries=150, sequenceid=313, filesize=12.0 K 2024-11-22T17:16:12,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/5c7516f7fc8241cf898f19e95771c198 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5c7516f7fc8241cf898f19e95771c198 2024-11-22T17:16:12,703 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5c7516f7fc8241cf898f19e95771c198, entries=150, sequenceid=313, filesize=12.0 K 2024-11-22T17:16:12,704 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2477a80617ad0e4131ef1992c2af963e in 1272ms, sequenceid=313, compaction requested=false 2024-11-22T17:16:12,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:12,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:12,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-22T17:16:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-22T17:16:12,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-22T17:16:12,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3410 sec 2024-11-22T17:16:12,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.3460 sec 2024-11-22T17:16:13,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:13,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-22T17:16:13,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:16:13,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:13,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:16:13,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:13,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:16:13,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:13,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227e898b538538495999a7e7e51c1baf84_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295773097/Put/seqid=0 2024-11-22T17:16:13,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742207_1383 (size=12454) 2024-11-22T17:16:13,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295833179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295833180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295833181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295833185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295833302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295833302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295833311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295833311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,516 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T17:16:13,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295833511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295833511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295833531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295833531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,561 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:13,564 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227e898b538538495999a7e7e51c1baf84_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227e898b538538495999a7e7e51c1baf84_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:13,565 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/d6eb1545231b4d71af506b9757b32012, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:13,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/d6eb1545231b4d71af506b9757b32012 is 175, key is test_row_0/A:col10/1732295773097/Put/seqid=0 2024-11-22T17:16:13,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742208_1384 (size=31255) 2024-11-22T17:16:13,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295833831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295833831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295833842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295833851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295833961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:13,968 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_0' in 2477a80617ad0e4131ef1992c2af963e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:13,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:13,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:13,969 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/d6eb1545231b4d71af506b9757b32012 2024-11-22T17:16:13,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/5e8a0ef112024c719f0ee10b0a1c1a76 is 50, key is test_row_0/B:col10/1732295773097/Put/seqid=0 2024-11-22T17:16:13,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742209_1385 (size=12301) 2024-11-22T17:16:14,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295834352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:14,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295834352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:14,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:14,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:14,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:14,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295834361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:14,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:14,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:14,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:14,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295834361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:14,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:14,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:14,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:14,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/5e8a0ef112024c719f0ee10b0a1c1a76 2024-11-22T17:16:14,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/4bde1899bee34ef88f13e5c10c1ca09e is 50, key is test_row_0/C:col10/1732295773097/Put/seqid=0 2024-11-22T17:16:14,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742210_1386 (size=12301) 2024-11-22T17:16:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T17:16:14,522 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:14,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:16:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-22T17:16:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T17:16:14,523 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:16:14,524 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:16:14,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:16:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T17:16:14,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-22T17:16:14,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:14,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:14,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:14,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:14,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:14,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/4bde1899bee34ef88f13e5c10c1ca09e 2024-11-22T17:16:14,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/d6eb1545231b4d71af506b9757b32012 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/d6eb1545231b4d71af506b9757b32012 2024-11-22T17:16:14,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/d6eb1545231b4d71af506b9757b32012, entries=150, sequenceid=333, filesize=30.5 K 2024-11-22T17:16:14,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/5e8a0ef112024c719f0ee10b0a1c1a76 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/5e8a0ef112024c719f0ee10b0a1c1a76 2024-11-22T17:16:14,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/5e8a0ef112024c719f0ee10b0a1c1a76, entries=150, sequenceid=333, filesize=12.0 K 2024-11-22T17:16:14,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/4bde1899bee34ef88f13e5c10c1ca09e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/4bde1899bee34ef88f13e5c10c1ca09e 2024-11-22T17:16:14,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/4bde1899bee34ef88f13e5c10c1ca09e, entries=150, sequenceid=333, filesize=12.0 K 2024-11-22T17:16:14,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 2477a80617ad0e4131ef1992c2af963e in 1676ms, sequenceid=333, compaction requested=true 2024-11-22T17:16:14,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:14,821 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:14,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:16:14,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:14,821 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:14,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:16:14,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:14,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:16:14,822 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:14,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:14,822 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/A is initiating minor compaction (all files) 2024-11-22T17:16:14,823 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/A in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:14,823 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/301a85fb85024c1ea1f8dbf4c8ffba65, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/195942a783f14e97bd60a6b479c1d02d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/d6eb1545231b4d71af506b9757b32012] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=92.2 K 2024-11-22T17:16:14,823 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:14,823 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/301a85fb85024c1ea1f8dbf4c8ffba65, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/195942a783f14e97bd60a6b479c1d02d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/d6eb1545231b4d71af506b9757b32012] 2024-11-22T17:16:14,824 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:14,824 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/B is initiating minor compaction (all files) 2024-11-22T17:16:14,824 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/B in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:14,824 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 301a85fb85024c1ea1f8dbf4c8ffba65, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732295768511 2024-11-22T17:16:14,824 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/99bc7d2a5d234aa7a80f939449583f6a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/c070da46e81f46a4ab1b8ecc5c123e14, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/5e8a0ef112024c719f0ee10b0a1c1a76] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=36.7 K 2024-11-22T17:16:14,824 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 195942a783f14e97bd60a6b479c1d02d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732295769715 2024-11-22T17:16:14,824 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 99bc7d2a5d234aa7a80f939449583f6a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732295768511 2024-11-22T17:16:14,824 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6eb1545231b4d71af506b9757b32012, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732295771921 2024-11-22T17:16:14,825 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting c070da46e81f46a4ab1b8ecc5c123e14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732295769715 2024-11-22T17:16:14,825 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e8a0ef112024c719f0ee10b0a1c1a76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732295771921 2024-11-22T17:16:14,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-22T17:16:14,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:14,829 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-22T17:16:14,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:16:14,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:14,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:16:14,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:14,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:16:14,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:14,836 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c183e33a3a544eb0bf4047c3f391d332_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295773160/Put/seqid=0 2024-11-22T17:16:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T17:16:14,849 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411223f326b46d50045f790901213c8015733_2477a80617ad0e4131ef1992c2af963e store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:14,850 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#B#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:14,850 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/6f7f4ab5e9404e8faf0ae7655715a265 is 50, key is test_row_0/B:col10/1732295773097/Put/seqid=0 2024-11-22T17:16:14,851 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411223f326b46d50045f790901213c8015733_2477a80617ad0e4131ef1992c2af963e, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:14,851 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223f326b46d50045f790901213c8015733_2477a80617ad0e4131ef1992c2af963e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:14,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742211_1387 (size=12454) 2024-11-22T17:16:14,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742212_1388 (size=13051) 2024-11-22T17:16:14,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742213_1389 (size=4469) 2024-11-22T17:16:14,862 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#A#compaction#322 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:14,862 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/6c198a52fbfb4e449cbbeb48434611ff is 175, key is test_row_0/A:col10/1732295773097/Put/seqid=0 2024-11-22T17:16:14,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742214_1390 (size=32005) 2024-11-22T17:16:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T17:16:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:15,260 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c183e33a3a544eb0bf4047c3f391d332_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c183e33a3a544eb0bf4047c3f391d332_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:15,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/4748a78f4f514bd4a970a95f5db5d5dd, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/4748a78f4f514bd4a970a95f5db5d5dd is 175, key is test_row_0/A:col10/1732295773160/Put/seqid=0 2024-11-22T17:16:15,265 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/6f7f4ab5e9404e8faf0ae7655715a265 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/6f7f4ab5e9404e8faf0ae7655715a265 2024-11-22T17:16:15,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742215_1391 (size=31255) 2024-11-22T17:16:15,270 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/B of 2477a80617ad0e4131ef1992c2af963e into 6f7f4ab5e9404e8faf0ae7655715a265(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:15,270 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:15,270 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/B, priority=13, startTime=1732295774821; duration=0sec 2024-11-22T17:16:15,270 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:15,270 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:B 2024-11-22T17:16:15,270 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:15,271 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:15,271 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/C is initiating minor compaction (all files) 2024-11-22T17:16:15,271 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/C in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:15,271 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/513eabf0846f4d319bff35a39f6a74f4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5c7516f7fc8241cf898f19e95771c198, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/4bde1899bee34ef88f13e5c10c1ca09e] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=36.7 K 2024-11-22T17:16:15,272 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 513eabf0846f4d319bff35a39f6a74f4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732295768511 2024-11-22T17:16:15,272 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c7516f7fc8241cf898f19e95771c198, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732295769715 2024-11-22T17:16:15,272 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bde1899bee34ef88f13e5c10c1ca09e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732295771921 2024-11-22T17:16:15,275 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/6c198a52fbfb4e449cbbeb48434611ff as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/6c198a52fbfb4e449cbbeb48434611ff 2024-11-22T17:16:15,279 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#C#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:15,279 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/f492d0e3ff77496b9acfa39034a1db55 is 50, key is test_row_0/C:col10/1732295773097/Put/seqid=0 2024-11-22T17:16:15,280 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/A of 2477a80617ad0e4131ef1992c2af963e into 6c198a52fbfb4e449cbbeb48434611ff(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:15,280 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:15,280 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/A, priority=13, startTime=1732295774821; duration=0sec 2024-11-22T17:16:15,280 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:15,280 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:A 2024-11-22T17:16:15,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742216_1392 (size=13051) 2024-11-22T17:16:15,288 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/f492d0e3ff77496b9acfa39034a1db55 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/f492d0e3ff77496b9acfa39034a1db55 2024-11-22T17:16:15,292 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/C of 2477a80617ad0e4131ef1992c2af963e into f492d0e3ff77496b9acfa39034a1db55(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:15,292 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:15,292 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/C, priority=13, startTime=1732295774822; duration=0sec 2024-11-22T17:16:15,292 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:15,292 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:C 2024-11-22T17:16:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:15,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. as already flushing 2024-11-22T17:16:15,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295835396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:15,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:15,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295835402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295835402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:15,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295835404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:15,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295835511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:15,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295835511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:15,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295835517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:15,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295835521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:15,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T17:16:15,666 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/4748a78f4f514bd4a970a95f5db5d5dd 2024-11-22T17:16:15,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/a21515797b034f92bb6f95de31719947 is 50, key is test_row_0/B:col10/1732295773160/Put/seqid=0 2024-11-22T17:16:15,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742217_1393 (size=12301) 2024-11-22T17:16:15,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295835721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:15,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295835721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:15,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:15,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295835731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:15,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295835731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:16,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58444 deadline: 1732295836041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58428 deadline: 1732295836041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:16,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295836051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:16,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:16,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295836051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:16,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:16,076 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/a21515797b034f92bb6f95de31719947 2024-11-22T17:16:16,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/392ab8f2cdab4bd1a5aee10912eacf3d is 50, key is test_row_0/C:col10/1732295773160/Put/seqid=0 2024-11-22T17:16:16,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742218_1394 (size=12301) 2024-11-22T17:16:16,086 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/392ab8f2cdab4bd1a5aee10912eacf3d 2024-11-22T17:16:16,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/4748a78f4f514bd4a970a95f5db5d5dd as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/4748a78f4f514bd4a970a95f5db5d5dd 2024-11-22T17:16:16,092 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/4748a78f4f514bd4a970a95f5db5d5dd, entries=150, sequenceid=348, filesize=30.5 K 2024-11-22T17:16:16,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/a21515797b034f92bb6f95de31719947 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/a21515797b034f92bb6f95de31719947 2024-11-22T17:16:16,096 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/a21515797b034f92bb6f95de31719947, entries=150, sequenceid=348, filesize=12.0 K 2024-11-22T17:16:16,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/392ab8f2cdab4bd1a5aee10912eacf3d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/392ab8f2cdab4bd1a5aee10912eacf3d 2024-11-22T17:16:16,100 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/392ab8f2cdab4bd1a5aee10912eacf3d, entries=150, sequenceid=348, filesize=12.0 K 2024-11-22T17:16:16,101 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2477a80617ad0e4131ef1992c2af963e in 1272ms, sequenceid=348, compaction requested=false 2024-11-22T17:16:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-22T17:16:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-22T17:16:16,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-22T17:16:16,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5780 sec 2024-11-22T17:16:16,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.5820 sec 2024-11-22T17:16:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:16,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T17:16:16,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:16:16,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:16,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:16:16,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:16,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:16:16,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:16,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112261bbdd8d102d4149bffff46bde49e613_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295775402/Put/seqid=0 2024-11-22T17:16:16,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742219_1395 (size=14994) 2024-11-22T17:16:16,597 INFO [Thread-1388 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1388. 2024-11-22T17:16:16,597 DEBUG [Thread-1388 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:16,597 DEBUG [Thread-1388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,597 DEBUG [Thread-1388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:16,598 INFO [Thread-1395 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1395. 2024-11-22T17:16:16,598 DEBUG [Thread-1395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:16,598 DEBUG [Thread-1395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,598 DEBUG [Thread-1395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:16,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:16,598 INFO [Thread-1386 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1386. 2024-11-22T17:16:16,598 DEBUG [Thread-1386 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:16,598 DEBUG [Thread-1386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295836588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:16,598 DEBUG [Thread-1386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,599 INFO [Thread-1399 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1399. 2024-11-22T17:16:16,599 DEBUG [Thread-1399 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:16,599 DEBUG [Thread-1399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,599 DEBUG [Thread-1399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:16,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:16,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:16,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295836597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:16,603 INFO [Thread-1401 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1401. 2024-11-22T17:16:16,603 DEBUG [Thread-1401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:16,603 DEBUG [Thread-1401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,603 DEBUG [Thread-1401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,603 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:16,609 INFO [Thread-1393 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1393. 2024-11-22T17:16:16,609 DEBUG [Thread-1393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:16,609 DEBUG [Thread-1393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,609 DEBUG [Thread-1393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:16,611 INFO [Thread-1397 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1397. 2024-11-22T17:16:16,611 DEBUG [Thread-1397 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:16,611 DEBUG [Thread-1397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,611 DEBUG [Thread-1397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:16,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:16,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T17:16:16,672 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:16,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295836701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:16,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:16,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295836711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:16,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295836912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:16,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295836922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:16,978 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:16,988 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112261bbdd8d102d4149bffff46bde49e613_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112261bbdd8d102d4149bffff46bde49e613_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:16,990 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/91829f7a66554dfd89786f78379978f8, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:16,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/91829f7a66554dfd89786f78379978f8 is 175, key is test_row_0/A:col10/1732295775402/Put/seqid=0 2024-11-22T17:16:16,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742220_1396 (size=39949) 2024-11-22T17:16:17,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295837221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:17,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295837231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:17,396 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/91829f7a66554dfd89786f78379978f8 2024-11-22T17:16:17,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/2d0f298f5bbd428098a71174ed0d7a4b is 50, key is test_row_0/B:col10/1732295775402/Put/seqid=0 2024-11-22T17:16:17,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742221_1397 (size=12301) 2024-11-22T17:16:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1732295837732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:17,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58460 deadline: 1732295837741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:17,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/2d0f298f5bbd428098a71174ed0d7a4b 2024-11-22T17:16:17,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/21507e32d2274e07a3c1f090c204b330 is 50, key is test_row_0/C:col10/1732295775402/Put/seqid=0 2024-11-22T17:16:17,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742222_1398 (size=12301) 2024-11-22T17:16:17,955 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/ab1f39e74bd54456a1f259fcdcf6e61c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5dfbab3d139948f6b66d774f3780a180, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/64c28e0a96ce4e24af031e0295d221c1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/6130a0dfc40447c1b51096374049e07f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/1046d1bd1317455e8795efa4d7f60818, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5ace3af83efd4af49410f4ec6b8e5fc1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/fa6c14cd26284343b00cb67f9164d9d9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/09a4f51f6523496682b556226a683727, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/72b07bdf84b1422da1f50ea5d07279d9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/4dede57c5d52498690aeccb2dca8ee34, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/f9056842f72046f899774e6e880e3287, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/bbe21a32b4424962844c5832efc9a850, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/88f85d47edd8438488c4b83c0e4185ef, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/32e0e1f56e7a4df5a278bd97b2caf765, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5fdab12892bc4f4ba74509e917d0e262, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/3202f3c3877a400f88643e86d575146f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/11e176a36ede49699d1a5b6fc83154ba, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/75187badf3674602b33de3833fcf5cac, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/b98d22e7a31948348a199e29178f60fe, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/301a85fb85024c1ea1f8dbf4c8ffba65, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/195942a783f14e97bd60a6b479c1d02d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/d6eb1545231b4d71af506b9757b32012] to archive 2024-11-22T17:16:17,957 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:16:17,960 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/ab1f39e74bd54456a1f259fcdcf6e61c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/ab1f39e74bd54456a1f259fcdcf6e61c 2024-11-22T17:16:17,963 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5dfbab3d139948f6b66d774f3780a180 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5dfbab3d139948f6b66d774f3780a180 2024-11-22T17:16:17,965 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/64c28e0a96ce4e24af031e0295d221c1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/64c28e0a96ce4e24af031e0295d221c1 2024-11-22T17:16:17,966 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/6130a0dfc40447c1b51096374049e07f to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/6130a0dfc40447c1b51096374049e07f 2024-11-22T17:16:17,967 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/1046d1bd1317455e8795efa4d7f60818 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/1046d1bd1317455e8795efa4d7f60818 2024-11-22T17:16:17,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5ace3af83efd4af49410f4ec6b8e5fc1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5ace3af83efd4af49410f4ec6b8e5fc1 2024-11-22T17:16:17,969 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/fa6c14cd26284343b00cb67f9164d9d9 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/fa6c14cd26284343b00cb67f9164d9d9 2024-11-22T17:16:17,970 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/09a4f51f6523496682b556226a683727 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/09a4f51f6523496682b556226a683727 2024-11-22T17:16:17,972 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/72b07bdf84b1422da1f50ea5d07279d9 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/72b07bdf84b1422da1f50ea5d07279d9 2024-11-22T17:16:17,973 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/4dede57c5d52498690aeccb2dca8ee34 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/4dede57c5d52498690aeccb2dca8ee34 2024-11-22T17:16:17,974 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/f9056842f72046f899774e6e880e3287 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/f9056842f72046f899774e6e880e3287 2024-11-22T17:16:17,975 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/bbe21a32b4424962844c5832efc9a850 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/bbe21a32b4424962844c5832efc9a850 2024-11-22T17:16:17,976 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/88f85d47edd8438488c4b83c0e4185ef to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/88f85d47edd8438488c4b83c0e4185ef 2024-11-22T17:16:17,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/32e0e1f56e7a4df5a278bd97b2caf765 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/32e0e1f56e7a4df5a278bd97b2caf765 2024-11-22T17:16:17,978 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5fdab12892bc4f4ba74509e917d0e262 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5fdab12892bc4f4ba74509e917d0e262 2024-11-22T17:16:17,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/3202f3c3877a400f88643e86d575146f to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/3202f3c3877a400f88643e86d575146f 2024-11-22T17:16:17,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/11e176a36ede49699d1a5b6fc83154ba to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/11e176a36ede49699d1a5b6fc83154ba 2024-11-22T17:16:17,980 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/75187badf3674602b33de3833fcf5cac to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/75187badf3674602b33de3833fcf5cac 2024-11-22T17:16:17,981 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/b98d22e7a31948348a199e29178f60fe to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/b98d22e7a31948348a199e29178f60fe 2024-11-22T17:16:17,982 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/301a85fb85024c1ea1f8dbf4c8ffba65 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/301a85fb85024c1ea1f8dbf4c8ffba65 2024-11-22T17:16:17,982 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/195942a783f14e97bd60a6b479c1d02d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/195942a783f14e97bd60a6b479c1d02d 2024-11-22T17:16:17,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/d6eb1545231b4d71af506b9757b32012 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/d6eb1545231b4d71af506b9757b32012 2024-11-22T17:16:17,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/d55ad39b211f4ed7b2687f3eefd799b1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/f3bd8eaa6e724f0eaf260512d1d3866e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8e3de9dd7d7540139c6342c11c32bc12, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/0152c1a305624d61acc08669ba4a46ca, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/ccae1428ef6a490680707cb74caed610, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b6e3b5e157ca4e799f73481a7616fa1e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/28a758c6a4f2456e9ea51042abdfeebf, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/0c507286ccbd4f0c828d7ac183e72dc4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/9c499625988a41cebf23ac58fb95df24, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/ec15b3247f12443e99e1676d0e2e651b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/a1219c799e7641118802fe0925039859, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b3db6e278ae54627871aec2f56c85b88, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8456d24ef7624072b72861f730a98aaf, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/1f86388675974107a2de37a594cc709a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/970300e115db4d91b9a0a7b205fc925a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/678b20a268b54beda0238c6616a1289b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/c18b2f0e35b44596a1cac2bbb9558a32, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b3b13a01c154452cb0d0cfe7689ec9e0, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/99bc7d2a5d234aa7a80f939449583f6a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/f76f35d9f4244caeb1f0ee2033322a51, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/c070da46e81f46a4ab1b8ecc5c123e14, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/5e8a0ef112024c719f0ee10b0a1c1a76] to archive 2024-11-22T17:16:17,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:16:17,988 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/d55ad39b211f4ed7b2687f3eefd799b1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/d55ad39b211f4ed7b2687f3eefd799b1 2024-11-22T17:16:17,988 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/f3bd8eaa6e724f0eaf260512d1d3866e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/f3bd8eaa6e724f0eaf260512d1d3866e 2024-11-22T17:16:17,989 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8e3de9dd7d7540139c6342c11c32bc12 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8e3de9dd7d7540139c6342c11c32bc12 2024-11-22T17:16:17,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/0152c1a305624d61acc08669ba4a46ca to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/0152c1a305624d61acc08669ba4a46ca 2024-11-22T17:16:17,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/ccae1428ef6a490680707cb74caed610 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/ccae1428ef6a490680707cb74caed610 2024-11-22T17:16:17,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58434 deadline: 1732295837991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:17,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b6e3b5e157ca4e799f73481a7616fa1e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b6e3b5e157ca4e799f73481a7616fa1e 2024-11-22T17:16:17,992 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_0' in 2477a80617ad0e4131ef1992c2af963e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:16:17,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/28a758c6a4f2456e9ea51042abdfeebf to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/28a758c6a4f2456e9ea51042abdfeebf 2024-11-22T17:16:17,994 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/0c507286ccbd4f0c828d7ac183e72dc4 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/0c507286ccbd4f0c828d7ac183e72dc4 2024-11-22T17:16:17,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/9c499625988a41cebf23ac58fb95df24 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/9c499625988a41cebf23ac58fb95df24 2024-11-22T17:16:17,996 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/ec15b3247f12443e99e1676d0e2e651b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/ec15b3247f12443e99e1676d0e2e651b 2024-11-22T17:16:17,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/a1219c799e7641118802fe0925039859 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/a1219c799e7641118802fe0925039859 2024-11-22T17:16:17,998 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b3db6e278ae54627871aec2f56c85b88 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b3db6e278ae54627871aec2f56c85b88 2024-11-22T17:16:17,999 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8456d24ef7624072b72861f730a98aaf to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8456d24ef7624072b72861f730a98aaf 2024-11-22T17:16:17,999 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/1f86388675974107a2de37a594cc709a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/1f86388675974107a2de37a594cc709a 2024-11-22T17:16:18,000 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/970300e115db4d91b9a0a7b205fc925a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/970300e115db4d91b9a0a7b205fc925a 2024-11-22T17:16:18,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/678b20a268b54beda0238c6616a1289b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/678b20a268b54beda0238c6616a1289b 2024-11-22T17:16:18,002 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/c18b2f0e35b44596a1cac2bbb9558a32 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/c18b2f0e35b44596a1cac2bbb9558a32 2024-11-22T17:16:18,003 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b3b13a01c154452cb0d0cfe7689ec9e0 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/b3b13a01c154452cb0d0cfe7689ec9e0 2024-11-22T17:16:18,003 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/99bc7d2a5d234aa7a80f939449583f6a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/99bc7d2a5d234aa7a80f939449583f6a 2024-11-22T17:16:18,004 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/f76f35d9f4244caeb1f0ee2033322a51 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/f76f35d9f4244caeb1f0ee2033322a51 2024-11-22T17:16:18,005 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/c070da46e81f46a4ab1b8ecc5c123e14 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/c070da46e81f46a4ab1b8ecc5c123e14 2024-11-22T17:16:18,006 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/5e8a0ef112024c719f0ee10b0a1c1a76 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/5e8a0ef112024c719f0ee10b0a1c1a76 2024-11-22T17:16:18,009 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/345ee4b0a3274c809f4fec557329dbf8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/cd0e787ad4d54e2985ba86d43593d82c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/157e182ebcda43df97e80c5c814e06ed, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/a0eb1106e0de41f1b7895174210b2fd3, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/21efe3e3476944d6b3e44fd057028e1d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/d2f196e758e9449c9174cab7324ff432, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5e8f42d15b6b42808d7eaf4d9a0d26d9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/72ef0019aae64ecfbd2899ea797cbaca, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/0138b0ed639b498883ef9b235e083fed, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5f445b2983e54ea1b3f0b0d157f652b5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/e84727d8d4714616944079f476b09080, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/4427d39e134b44d6bdafa53e6437d91f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/a16776b985e04939a4efa21cf1281254, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/c36f34ef5db3492ebc16d5d0fd97a2ae, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/c0fb6fce110e4c839826fbd9321dc4be, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/8a5286532f8e479799f1bc2d6bfb9b55, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/549eada901a24d40a59734ff3f77c2c9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/46e5522672094259852edf9bf5b51559, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/513eabf0846f4d319bff35a39f6a74f4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/d32aafa01a06479392c02abae2d57d5c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5c7516f7fc8241cf898f19e95771c198, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/4bde1899bee34ef88f13e5c10c1ca09e] to archive 2024-11-22T17:16:18,010 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:16:18,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/345ee4b0a3274c809f4fec557329dbf8 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/345ee4b0a3274c809f4fec557329dbf8 2024-11-22T17:16:18,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/cd0e787ad4d54e2985ba86d43593d82c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/cd0e787ad4d54e2985ba86d43593d82c 2024-11-22T17:16:18,013 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/157e182ebcda43df97e80c5c814e06ed to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/157e182ebcda43df97e80c5c814e06ed 2024-11-22T17:16:18,014 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/a0eb1106e0de41f1b7895174210b2fd3 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/a0eb1106e0de41f1b7895174210b2fd3 2024-11-22T17:16:18,015 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/21efe3e3476944d6b3e44fd057028e1d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/21efe3e3476944d6b3e44fd057028e1d 2024-11-22T17:16:18,016 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/d2f196e758e9449c9174cab7324ff432 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/d2f196e758e9449c9174cab7324ff432 2024-11-22T17:16:18,017 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5e8f42d15b6b42808d7eaf4d9a0d26d9 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5e8f42d15b6b42808d7eaf4d9a0d26d9 2024-11-22T17:16:18,018 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/72ef0019aae64ecfbd2899ea797cbaca to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/72ef0019aae64ecfbd2899ea797cbaca 2024-11-22T17:16:18,019 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/0138b0ed639b498883ef9b235e083fed to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/0138b0ed639b498883ef9b235e083fed 2024-11-22T17:16:18,020 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5f445b2983e54ea1b3f0b0d157f652b5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5f445b2983e54ea1b3f0b0d157f652b5 2024-11-22T17:16:18,021 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/e84727d8d4714616944079f476b09080 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/e84727d8d4714616944079f476b09080 2024-11-22T17:16:18,022 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/4427d39e134b44d6bdafa53e6437d91f to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/4427d39e134b44d6bdafa53e6437d91f 2024-11-22T17:16:18,023 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/a16776b985e04939a4efa21cf1281254 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/a16776b985e04939a4efa21cf1281254 2024-11-22T17:16:18,024 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/c36f34ef5db3492ebc16d5d0fd97a2ae to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/c36f34ef5db3492ebc16d5d0fd97a2ae 2024-11-22T17:16:18,025 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/c0fb6fce110e4c839826fbd9321dc4be to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/c0fb6fce110e4c839826fbd9321dc4be 2024-11-22T17:16:18,026 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/8a5286532f8e479799f1bc2d6bfb9b55 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/8a5286532f8e479799f1bc2d6bfb9b55 2024-11-22T17:16:18,027 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/549eada901a24d40a59734ff3f77c2c9 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/549eada901a24d40a59734ff3f77c2c9 2024-11-22T17:16:18,028 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/46e5522672094259852edf9bf5b51559 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/46e5522672094259852edf9bf5b51559 2024-11-22T17:16:18,029 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/513eabf0846f4d319bff35a39f6a74f4 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/513eabf0846f4d319bff35a39f6a74f4 2024-11-22T17:16:18,030 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/d32aafa01a06479392c02abae2d57d5c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/d32aafa01a06479392c02abae2d57d5c 2024-11-22T17:16:18,031 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5c7516f7fc8241cf898f19e95771c198 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/5c7516f7fc8241cf898f19e95771c198 2024-11-22T17:16:18,032 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/4bde1899bee34ef88f13e5c10c1ca09e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/4bde1899bee34ef88f13e5c10c1ca09e 2024-11-22T17:16:18,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/21507e32d2274e07a3c1f090c204b330 2024-11-22T17:16:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/91829f7a66554dfd89786f78379978f8 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/91829f7a66554dfd89786f78379978f8 2024-11-22T17:16:18,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/91829f7a66554dfd89786f78379978f8, entries=200, sequenceid=369, filesize=39.0 K 2024-11-22T17:16:18,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/2d0f298f5bbd428098a71174ed0d7a4b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/2d0f298f5bbd428098a71174ed0d7a4b 2024-11-22T17:16:18,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/2d0f298f5bbd428098a71174ed0d7a4b, entries=150, sequenceid=369, filesize=12.0 K 2024-11-22T17:16:18,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/21507e32d2274e07a3c1f090c204b330 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/21507e32d2274e07a3c1f090c204b330 2024-11-22T17:16:18,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/21507e32d2274e07a3c1f090c204b330, entries=150, sequenceid=369, filesize=12.0 K 2024-11-22T17:16:18,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2477a80617ad0e4131ef1992c2af963e in 1693ms, sequenceid=369, compaction requested=true 2024-11-22T17:16:18,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:18,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:16:18,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:18,261 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:18,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:16:18,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:18,261 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:18,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2477a80617ad0e4131ef1992c2af963e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:16:18,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:18,262 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:18,262 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:18,262 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/A is initiating minor compaction (all files) 2024-11-22T17:16:18,262 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/B is initiating minor compaction (all files) 2024-11-22T17:16:18,262 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/B in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:18,262 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/A in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:18,262 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/6f7f4ab5e9404e8faf0ae7655715a265, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/a21515797b034f92bb6f95de31719947, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/2d0f298f5bbd428098a71174ed0d7a4b] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=36.8 K 2024-11-22T17:16:18,262 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/6c198a52fbfb4e449cbbeb48434611ff, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/4748a78f4f514bd4a970a95f5db5d5dd, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/91829f7a66554dfd89786f78379978f8] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=100.8 K 2024-11-22T17:16:18,262 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:18,262 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/6c198a52fbfb4e449cbbeb48434611ff, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/4748a78f4f514bd4a970a95f5db5d5dd, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/91829f7a66554dfd89786f78379978f8] 2024-11-22T17:16:18,263 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f7f4ab5e9404e8faf0ae7655715a265, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732295771921 2024-11-22T17:16:18,263 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c198a52fbfb4e449cbbeb48434611ff, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732295771921 2024-11-22T17:16:18,263 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a21515797b034f92bb6f95de31719947, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732295773160 2024-11-22T17:16:18,263 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4748a78f4f514bd4a970a95f5db5d5dd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732295773160 2024-11-22T17:16:18,263 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d0f298f5bbd428098a71174ed0d7a4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732295775394 2024-11-22T17:16:18,264 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91829f7a66554dfd89786f78379978f8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732295775394 2024-11-22T17:16:18,276 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:18,277 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#B#compaction#330 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:18,277 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/51a6b0426cb34d42a16ec8d90d9940b9 is 50, key is test_row_0/B:col10/1732295775402/Put/seqid=0 2024-11-22T17:16:18,279 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122d6db102321174ebeab721ff66507ca49_2477a80617ad0e4131ef1992c2af963e store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:18,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742223_1399 (size=12404) 2024-11-22T17:16:18,300 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122d6db102321174ebeab721ff66507ca49_2477a80617ad0e4131ef1992c2af963e, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:18,301 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d6db102321174ebeab721ff66507ca49_2477a80617ad0e4131ef1992c2af963e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:18,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742224_1400 (size=4469) 2024-11-22T17:16:18,685 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/51a6b0426cb34d42a16ec8d90d9940b9 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/51a6b0426cb34d42a16ec8d90d9940b9 2024-11-22T17:16:18,689 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/B of 2477a80617ad0e4131ef1992c2af963e into 51a6b0426cb34d42a16ec8d90d9940b9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:18,689 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:18,689 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/B, priority=13, startTime=1732295778261; duration=0sec 2024-11-22T17:16:18,690 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:18,690 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:B 2024-11-22T17:16:18,690 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:18,690 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:18,691 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 2477a80617ad0e4131ef1992c2af963e/C is initiating minor compaction (all files) 2024-11-22T17:16:18,691 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2477a80617ad0e4131ef1992c2af963e/C in TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:18,691 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/f492d0e3ff77496b9acfa39034a1db55, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/392ab8f2cdab4bd1a5aee10912eacf3d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/21507e32d2274e07a3c1f090c204b330] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp, totalSize=36.8 K 2024-11-22T17:16:18,691 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting f492d0e3ff77496b9acfa39034a1db55, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732295771921 2024-11-22T17:16:18,691 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 392ab8f2cdab4bd1a5aee10912eacf3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732295773160 2024-11-22T17:16:18,691 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 21507e32d2274e07a3c1f090c204b330, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732295775394 2024-11-22T17:16:18,700 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#C#compaction#332 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:18,701 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/8b9140a8767a4d3a989cf883796884fb is 50, key is test_row_0/C:col10/1732295775402/Put/seqid=0 2024-11-22T17:16:18,705 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2477a80617ad0e4131ef1992c2af963e#A#compaction#331 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:18,706 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5b300e6f9cd146aabc3024dd2791b5ba is 175, key is test_row_0/A:col10/1732295775402/Put/seqid=0 2024-11-22T17:16:18,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742225_1401 (size=12404) 2024-11-22T17:16:18,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742226_1402 (size=31358) 2024-11-22T17:16:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:18,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-22T17:16:18,753 INFO [Thread-1384 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1384. 2024-11-22T17:16:18,753 DEBUG [Thread-1384 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:18,753 DEBUG [Thread-1384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:18,753 DEBUG [Thread-1384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:18,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:16:18,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:18,753 INFO [Thread-1390 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1390. 2024-11-22T17:16:18,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:18,753 DEBUG [Thread-1390 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:18,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:16:18,753 DEBUG [Thread-1390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:18,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:18,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:16:18,753 DEBUG [Thread-1390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:18,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:18,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122136b2cf6ffe749ec8eddb75dcc67ea67_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295776596/Put/seqid=0 2024-11-22T17:16:18,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742227_1403 (size=12454) 2024-11-22T17:16:19,115 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/8b9140a8767a4d3a989cf883796884fb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/8b9140a8767a4d3a989cf883796884fb 2024-11-22T17:16:19,116 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/5b300e6f9cd146aabc3024dd2791b5ba as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5b300e6f9cd146aabc3024dd2791b5ba 2024-11-22T17:16:19,121 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/C of 2477a80617ad0e4131ef1992c2af963e into 8b9140a8767a4d3a989cf883796884fb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:19,121 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2477a80617ad0e4131ef1992c2af963e/A of 2477a80617ad0e4131ef1992c2af963e into 5b300e6f9cd146aabc3024dd2791b5ba(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:19,121 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:19,121 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:19,121 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/A, priority=13, startTime=1732295778260; duration=0sec 2024-11-22T17:16:19,121 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., storeName=2477a80617ad0e4131ef1992c2af963e/C, priority=13, startTime=1732295778261; duration=0sec 2024-11-22T17:16:19,121 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:19,121 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:19,121 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:A 2024-11-22T17:16:19,121 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2477a80617ad0e4131ef1992c2af963e:C 2024-11-22T17:16:19,165 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:19,170 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122136b2cf6ffe749ec8eddb75dcc67ea67_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122136b2cf6ffe749ec8eddb75dcc67ea67_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:19,171 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/a9b21e8af010430ea540e0217ba46953, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:19,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/a9b21e8af010430ea540e0217ba46953 is 175, key is test_row_0/A:col10/1732295776596/Put/seqid=0 2024-11-22T17:16:19,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742228_1404 (size=31255) 2024-11-22T17:16:19,578 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/a9b21e8af010430ea540e0217ba46953 2024-11-22T17:16:19,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/8289e4016a3e4d1498fa14e890c92bc3 is 50, key is test_row_0/B:col10/1732295776596/Put/seqid=0 2024-11-22T17:16:19,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742229_1405 (size=12301) 2024-11-22T17:16:19,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/8289e4016a3e4d1498fa14e890c92bc3 2024-11-22T17:16:20,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/de789d086dfc41f1be85e36200292ec9 is 50, key is test_row_0/C:col10/1732295776596/Put/seqid=0 2024-11-22T17:16:20,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742230_1406 (size=12301) 2024-11-22T17:16:20,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/de789d086dfc41f1be85e36200292ec9 2024-11-22T17:16:20,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/a9b21e8af010430ea540e0217ba46953 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/a9b21e8af010430ea540e0217ba46953 2024-11-22T17:16:20,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/a9b21e8af010430ea540e0217ba46953, entries=150, sequenceid=386, filesize=30.5 K 2024-11-22T17:16:20,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/8289e4016a3e4d1498fa14e890c92bc3 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8289e4016a3e4d1498fa14e890c92bc3 2024-11-22T17:16:20,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8289e4016a3e4d1498fa14e890c92bc3, entries=150, sequenceid=386, filesize=12.0 K 2024-11-22T17:16:20,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/de789d086dfc41f1be85e36200292ec9 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/de789d086dfc41f1be85e36200292ec9 2024-11-22T17:16:20,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/de789d086dfc41f1be85e36200292ec9, entries=150, sequenceid=386, filesize=12.0 K 2024-11-22T17:16:20,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 2477a80617ad0e4131ef1992c2af963e in 1677ms, sequenceid=386, compaction requested=false 2024-11-22T17:16:20,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2477a80617ad0e4131ef1992c2af963e: 2024-11-22T17:16:28,095 INFO [Thread-1382 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1382. 2024-11-22T17:16:28,095 DEBUG [Thread-1382 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:28,095 DEBUG [Thread-1382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:28,095 DEBUG [Thread-1382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:28,095 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1084 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3251 rows 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1094 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3254 rows 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1090 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3238 rows 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1092 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3245 rows 2024-11-22T17:16:28,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1098 2024-11-22T17:16:28,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3266 rows 2024-11-22T17:16:28,097 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T17:16:28,097 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T17:16:28,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:28,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:28,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:28,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T17:16:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T17:16:28,104 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295788104"}]},"ts":"1732295788104"} 2024-11-22T17:16:28,106 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T17:16:28,106 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T17:16:28,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T17:16:28,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, UNASSIGN}] 2024-11-22T17:16:28,109 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, UNASSIGN 2024-11-22T17:16:28,109 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=2477a80617ad0e4131ef1992c2af963e, regionState=CLOSING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:28,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, UNASSIGN because future has completed 2024-11-22T17:16:28,111 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T17:16:28,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:16:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T17:16:28,263 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:28,263 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T17:16:28,263 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 2477a80617ad0e4131ef1992c2af963e, disabling compactions & flushes 2024-11-22T17:16:28,263 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:28,263 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:28,263 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. after waiting 0 ms 2024-11-22T17:16:28,263 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:28,264 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 2477a80617ad0e4131ef1992c2af963e 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-22T17:16:28,264 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=A 2024-11-22T17:16:28,264 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:28,264 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=B 2024-11-22T17:16:28,264 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:28,264 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2477a80617ad0e4131ef1992c2af963e, store=C 2024-11-22T17:16:28,264 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:28,268 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122661f2bd5f24347a9bb20545b10692513_2477a80617ad0e4131ef1992c2af963e is 50, key is test_row_0/A:col10/1732295788092/Put/seqid=0 2024-11-22T17:16:28,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742231_1407 (size=7374) 2024-11-22T17:16:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T17:16:28,672 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:28,678 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122661f2bd5f24347a9bb20545b10692513_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122661f2bd5f24347a9bb20545b10692513_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:28,679 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/ffc1421acd34423d941eb26df1b641a3, store: [table=TestAcidGuarantees family=A region=2477a80617ad0e4131ef1992c2af963e] 2024-11-22T17:16:28,680 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/ffc1421acd34423d941eb26df1b641a3 is 175, key is test_row_0/A:col10/1732295788092/Put/seqid=0 2024-11-22T17:16:28,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742232_1408 (size=13865) 2024-11-22T17:16:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T17:16:29,086 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/ffc1421acd34423d941eb26df1b641a3 2024-11-22T17:16:29,101 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/24d8b20f5dc44e6199b64ea125be4fc7 is 50, key is test_row_0/B:col10/1732295788092/Put/seqid=0 2024-11-22T17:16:29,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742233_1409 (size=7415) 2024-11-22T17:16:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T17:16:29,506 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/24d8b20f5dc44e6199b64ea125be4fc7 2024-11-22T17:16:29,518 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/8968c0b28d5642baad0d780b71468bae is 50, key is test_row_0/C:col10/1732295788092/Put/seqid=0 2024-11-22T17:16:29,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742234_1410 (size=7415) 2024-11-22T17:16:29,923 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/8968c0b28d5642baad0d780b71468bae 2024-11-22T17:16:29,928 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/A/ffc1421acd34423d941eb26df1b641a3 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/ffc1421acd34423d941eb26df1b641a3 2024-11-22T17:16:29,933 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/ffc1421acd34423d941eb26df1b641a3, entries=50, sequenceid=392, filesize=13.5 K 2024-11-22T17:16:29,934 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/B/24d8b20f5dc44e6199b64ea125be4fc7 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/24d8b20f5dc44e6199b64ea125be4fc7 2024-11-22T17:16:29,939 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/24d8b20f5dc44e6199b64ea125be4fc7, entries=50, sequenceid=392, filesize=7.2 K 2024-11-22T17:16:29,940 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/.tmp/C/8968c0b28d5642baad0d780b71468bae as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/8968c0b28d5642baad0d780b71468bae 2024-11-22T17:16:29,943 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/8968c0b28d5642baad0d780b71468bae, entries=50, sequenceid=392, filesize=7.2 K 2024-11-22T17:16:29,944 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 2477a80617ad0e4131ef1992c2af963e in 1681ms, sequenceid=392, compaction requested=true 2024-11-22T17:16:29,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/6c198a52fbfb4e449cbbeb48434611ff, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/4748a78f4f514bd4a970a95f5db5d5dd, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/91829f7a66554dfd89786f78379978f8] to archive 2024-11-22T17:16:29,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:16:29,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/6c198a52fbfb4e449cbbeb48434611ff to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/6c198a52fbfb4e449cbbeb48434611ff 2024-11-22T17:16:29,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/4748a78f4f514bd4a970a95f5db5d5dd to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/4748a78f4f514bd4a970a95f5db5d5dd 2024-11-22T17:16:29,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/91829f7a66554dfd89786f78379978f8 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/91829f7a66554dfd89786f78379978f8 2024-11-22T17:16:29,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/6f7f4ab5e9404e8faf0ae7655715a265, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/a21515797b034f92bb6f95de31719947, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/2d0f298f5bbd428098a71174ed0d7a4b] to archive 2024-11-22T17:16:29,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:16:29,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/6f7f4ab5e9404e8faf0ae7655715a265 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/6f7f4ab5e9404e8faf0ae7655715a265 2024-11-22T17:16:29,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/a21515797b034f92bb6f95de31719947 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/a21515797b034f92bb6f95de31719947 2024-11-22T17:16:29,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/2d0f298f5bbd428098a71174ed0d7a4b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/2d0f298f5bbd428098a71174ed0d7a4b 2024-11-22T17:16:29,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/f492d0e3ff77496b9acfa39034a1db55, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/392ab8f2cdab4bd1a5aee10912eacf3d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/21507e32d2274e07a3c1f090c204b330] to archive 2024-11-22T17:16:29,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:16:29,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/f492d0e3ff77496b9acfa39034a1db55 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/f492d0e3ff77496b9acfa39034a1db55 2024-11-22T17:16:29,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/392ab8f2cdab4bd1a5aee10912eacf3d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/392ab8f2cdab4bd1a5aee10912eacf3d 2024-11-22T17:16:29,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/21507e32d2274e07a3c1f090c204b330 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/21507e32d2274e07a3c1f090c204b330 2024-11-22T17:16:29,961 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/recovered.edits/395.seqid, newMaxSeqId=395, maxSeqId=4 2024-11-22T17:16:29,961 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. 2024-11-22T17:16:29,961 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 2477a80617ad0e4131ef1992c2af963e: Waiting for close lock at 1732295788263Running coprocessor pre-close hooks at 1732295788263Disabling compacts and flushes for region at 1732295788263Disabling writes for close at 1732295788263Obtaining lock to block concurrent updates at 1732295788264 (+1 ms)Preparing flush snapshotting stores in 2477a80617ad0e4131ef1992c2af963e at 1732295788264Finished memstore snapshotting TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1732295788264Flushing stores of TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e. at 1732295788264Flushing 2477a80617ad0e4131ef1992c2af963e/A: creating writer at 1732295788265 (+1 ms)Flushing mob file 2477a80617ad0e4131ef1992c2af963e/A: appending metadata at 1732295788268 (+3 ms)Flushing mob file 2477a80617ad0e4131ef1992c2af963e/A: closing flushed file at 1732295788268Flushing 2477a80617ad0e4131ef1992c2af963e/A: appending metadata at 1732295788679 (+411 ms)Flushing 2477a80617ad0e4131ef1992c2af963e/A: closing flushed file at 1732295788679Flushing 2477a80617ad0e4131ef1992c2af963e/B: creating writer at 1732295789095 (+416 ms)Flushing 2477a80617ad0e4131ef1992c2af963e/B: appending metadata at 1732295789100 (+5 ms)Flushing 2477a80617ad0e4131ef1992c2af963e/B: closing flushed file at 1732295789100Flushing 2477a80617ad0e4131ef1992c2af963e/C: creating writer at 1732295789514 (+414 ms)Flushing 2477a80617ad0e4131ef1992c2af963e/C: appending metadata at 1732295789518 (+4 ms)Flushing 2477a80617ad0e4131ef1992c2af963e/C: closing flushed file at 1732295789518Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@feb42c7: reopening flushed file at 1732295789927 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@128bc982: reopening flushed file at 1732295789933 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70e23be9: reopening flushed file at 1732295789939 (+6 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 2477a80617ad0e4131ef1992c2af963e in 1681ms, sequenceid=392, compaction requested=true at 1732295789944 (+5 ms)Writing region close event to WAL at 1732295789958 (+14 ms)Running coprocessor post-close hooks at 1732295789961 (+3 ms)Closed at 1732295789961 2024-11-22T17:16:29,963 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:29,964 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=2477a80617ad0e4131ef1992c2af963e, regionState=CLOSED 2024-11-22T17:16:29,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:16:29,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-22T17:16:29,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 2477a80617ad0e4131ef1992c2af963e, server=e61df7af61d2,42253,1732295655623 in 1.8550 sec 2024-11-22T17:16:29,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-22T17:16:29,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2477a80617ad0e4131ef1992c2af963e, UNASSIGN in 1.8590 sec 2024-11-22T17:16:29,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-22T17:16:29,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8630 sec 2024-11-22T17:16:29,971 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295789971"}]},"ts":"1732295789971"} 2024-11-22T17:16:29,972 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T17:16:29,972 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T17:16:29,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8730 sec 2024-11-22T17:16:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T17:16:30,252 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:30,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T17:16:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:30,256 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T17:16:30,257 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:30,262 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,266 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/recovered.edits] 2024-11-22T17:16:30,269 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5b300e6f9cd146aabc3024dd2791b5ba to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/5b300e6f9cd146aabc3024dd2791b5ba 2024-11-22T17:16:30,270 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/a9b21e8af010430ea540e0217ba46953 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/a9b21e8af010430ea540e0217ba46953 2024-11-22T17:16:30,271 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/ffc1421acd34423d941eb26df1b641a3 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/A/ffc1421acd34423d941eb26df1b641a3 2024-11-22T17:16:30,273 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/24d8b20f5dc44e6199b64ea125be4fc7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/24d8b20f5dc44e6199b64ea125be4fc7 2024-11-22T17:16:30,274 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/51a6b0426cb34d42a16ec8d90d9940b9 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/51a6b0426cb34d42a16ec8d90d9940b9 2024-11-22T17:16:30,275 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8289e4016a3e4d1498fa14e890c92bc3 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/B/8289e4016a3e4d1498fa14e890c92bc3 2024-11-22T17:16:30,277 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/8968c0b28d5642baad0d780b71468bae to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/8968c0b28d5642baad0d780b71468bae 2024-11-22T17:16:30,277 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/8b9140a8767a4d3a989cf883796884fb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/8b9140a8767a4d3a989cf883796884fb 2024-11-22T17:16:30,278 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/de789d086dfc41f1be85e36200292ec9 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/C/de789d086dfc41f1be85e36200292ec9 2024-11-22T17:16:30,280 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/recovered.edits/395.seqid to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e/recovered.edits/395.seqid 2024-11-22T17:16:30,280 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,280 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T17:16:30,280 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T17:16:30,281 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T17:16:30,283 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112201215bc84719406d953229e137b56678_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112201215bc84719406d953229e137b56678_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,284 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122136b2cf6ffe749ec8eddb75dcc67ea67_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122136b2cf6ffe749ec8eddb75dcc67ea67_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,285 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411221beb189c180743e5b6359468d01ba68b_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411221beb189c180743e5b6359468d01ba68b_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,286 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112220b62df0c7ef471b84fc97d489eab22e_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112220b62df0c7ef471b84fc97d489eab22e_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,287 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222e3933f68a1a4e63859c85ff8c33b6e1_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222e3933f68a1a4e63859c85ff8c33b6e1_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,287 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122476ef6d4e0a5410d9b1b3772c28fbe28_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122476ef6d4e0a5410d9b1b3772c28fbe28_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,288 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112261bbdd8d102d4149bffff46bde49e613_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112261bbdd8d102d4149bffff46bde49e613_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,289 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122661f2bd5f24347a9bb20545b10692513_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122661f2bd5f24347a9bb20545b10692513_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,290 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226b7916c308f64022a79b536d2d283962_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226b7916c308f64022a79b536d2d283962_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,291 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226fbd56000d9448b3834541c63901831e_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226fbd56000d9448b3834541c63901831e_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,292 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112279ab66ea535b44a9864dd3cb7a2f112a_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112279ab66ea535b44a9864dd3cb7a2f112a_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,293 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227e898b538538495999a7e7e51c1baf84_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227e898b538538495999a7e7e51c1baf84_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,294 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228b978d7da8e54371bd9458106d5282b9_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228b978d7da8e54371bd9458106d5282b9_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,294 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122959a11dcb9d24fce9e95764599f1105f_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122959a11dcb9d24fce9e95764599f1105f_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,295 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a49df1881765466aa3334fa37ff3ad90_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a49df1881765466aa3334fa37ff3ad90_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,296 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c183e33a3a544eb0bf4047c3f391d332_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c183e33a3a544eb0bf4047c3f391d332_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,297 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e4022fe039eb42eb919016bc1b699501_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e4022fe039eb42eb919016bc1b699501_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,298 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122eae666db719946c7bcfde22a77ae1367_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122eae666db719946c7bcfde22a77ae1367_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,299 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122eb46bb19c0cd4e1b8c83e9138af18f86_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122eb46bb19c0cd4e1b8c83e9138af18f86_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,300 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fa6b3c2bc4e0413ba49d24b06154f2b0_2477a80617ad0e4131ef1992c2af963e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fa6b3c2bc4e0413ba49d24b06154f2b0_2477a80617ad0e4131ef1992c2af963e 2024-11-22T17:16:30,301 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T17:16:30,303 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:30,305 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T17:16:30,307 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T17:16:30,308 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:30,308 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T17:16:30,308 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732295790308"}]},"ts":"9223372036854775807"} 2024-11-22T17:16:30,310 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T17:16:30,310 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2477a80617ad0e4131ef1992c2af963e, NAME => 'TestAcidGuarantees,,1732295753429.2477a80617ad0e4131ef1992c2af963e.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T17:16:30,310 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T17:16:30,310 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732295790310"}]},"ts":"9223372036854775807"} 2024-11-22T17:16:30,312 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T17:16:30,313 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:30,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-11-22T17:16:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T17:16:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T17:16:30,365 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:30,373 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 249), OpenFileDescriptor=445 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=320 (was 335), ProcessCount=11 (was 11), AvailableMemoryMB=2007 (was 2259) 2024-11-22T17:16:30,383 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=320, ProcessCount=11, AvailableMemoryMB=2007 2024-11-22T17:16:30,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T17:16:30,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T17:16:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:30,388 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T17:16:30,388 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:30,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-22T17:16:30,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-22T17:16:30,389 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T17:16:30,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742235_1411 (size=1061) 2024-11-22T17:16:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-22T17:16:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-22T17:16:30,803 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8 2024-11-22T17:16:30,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742236_1412 (size=53) 2024-11-22T17:16:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-22T17:16:31,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:16:31,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5768c661d3c58ac27b37c9e997559f2f, disabling compactions & flushes 2024-11-22T17:16:31,213 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:31,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:31,214 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. after waiting 0 ms 2024-11-22T17:16:31,214 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:31,214 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:31,214 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5768c661d3c58ac27b37c9e997559f2f: Waiting for close lock at 1732295791213Disabling compacts and flushes for region at 1732295791213Disabling writes for close at 1732295791214 (+1 ms)Writing region close event to WAL at 1732295791214Closed at 1732295791214 2024-11-22T17:16:31,216 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T17:16:31,217 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732295791216"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732295791216"}]},"ts":"1732295791216"} 2024-11-22T17:16:31,220 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T17:16:31,222 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T17:16:31,222 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295791222"}]},"ts":"1732295791222"} 2024-11-22T17:16:31,224 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T17:16:31,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5768c661d3c58ac27b37c9e997559f2f, ASSIGN}] 2024-11-22T17:16:31,225 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5768c661d3c58ac27b37c9e997559f2f, ASSIGN 2024-11-22T17:16:31,226 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5768c661d3c58ac27b37c9e997559f2f, ASSIGN; state=OFFLINE, location=e61df7af61d2,42253,1732295655623; forceNewPlan=false, retain=false 2024-11-22T17:16:31,376 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=5768c661d3c58ac27b37c9e997559f2f, regionState=OPENING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:31,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5768c661d3c58ac27b37c9e997559f2f, ASSIGN because future has completed 2024-11-22T17:16:31,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:16:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-22T17:16:31,535 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:31,535 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} 2024-11-22T17:16:31,537 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:31,537 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:16:31,538 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:31,538 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:31,540 INFO [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:31,541 INFO [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:16:31,542 INFO [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5768c661d3c58ac27b37c9e997559f2f columnFamilyName A 2024-11-22T17:16:31,542 DEBUG [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:31,542 INFO [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] regionserver.HStore(327): Store=5768c661d3c58ac27b37c9e997559f2f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:16:31,542 INFO [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:31,543 INFO [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:16:31,544 INFO [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5768c661d3c58ac27b37c9e997559f2f columnFamilyName B 2024-11-22T17:16:31,544 DEBUG [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:31,544 INFO [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] regionserver.HStore(327): Store=5768c661d3c58ac27b37c9e997559f2f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:16:31,544 INFO [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:31,545 INFO [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:16:31,546 INFO [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5768c661d3c58ac27b37c9e997559f2f columnFamilyName C 2024-11-22T17:16:31,546 DEBUG [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:31,546 INFO [StoreOpener-5768c661d3c58ac27b37c9e997559f2f-1 {}] regionserver.HStore(327): Store=5768c661d3c58ac27b37c9e997559f2f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:16:31,546 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:31,546 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:31,547 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:31,548 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:31,549 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:31,549 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:31,550 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T17:16:31,551 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:31,554 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T17:16:31,555 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 5768c661d3c58ac27b37c9e997559f2f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74844194, jitterRate=0.11526539921760559}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T17:16:31,555 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:31,556 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 5768c661d3c58ac27b37c9e997559f2f: Running coprocessor pre-open hook at 1732295791538Writing region info on filesystem at 1732295791538Initializing all the Stores at 1732295791539 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295791539Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295791540 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295791540Cleaning up temporary data from old regions at 1732295791549 (+9 ms)Running coprocessor post-open hooks at 1732295791555 (+6 ms)Region opened successfully at 1732295791556 (+1 ms) 2024-11-22T17:16:31,557 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., pid=116, masterSystemTime=1732295791530 2024-11-22T17:16:31,559 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:31,559 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:31,559 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=5768c661d3c58ac27b37c9e997559f2f, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:31,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:16:31,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-22T17:16:31,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 in 184 msec 2024-11-22T17:16:31,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-22T17:16:31,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5768c661d3c58ac27b37c9e997559f2f, ASSIGN in 341 msec 2024-11-22T17:16:31,568 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T17:16:31,568 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295791568"}]},"ts":"1732295791568"} 2024-11-22T17:16:31,570 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T17:16:31,571 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T17:16:31,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-22T17:16:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-22T17:16:32,543 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:32,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277faa65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:16:32,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:16:32,549 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:16:32,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:16:32,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:16:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a23ca61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:16:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:16:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:32,552 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:16:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56a4c4b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,553 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:16:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:16:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:16:32,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:16:32,556 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T17:16:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is e61df7af61d2,45693,1732295654407 2024-11-22T17:16:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4f5b0292 2024-11-22T17:16:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T17:16:32,558 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T17:16:32,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@269ecf23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:16:32,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:16:32,560 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:16:32,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:16:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:16:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d26af7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:16:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:16:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:32,562 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:16:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d251fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695dbdde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:16:32,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:16:32,564 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:16:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:16:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:16:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@795e4e48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:16:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:16:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:32,566 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:16:32,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb936cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594674bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:16:32,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:16:32,568 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:16:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:16:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:16:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43bb0e4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:16:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:16:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:32,570 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:16:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bf5f197, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119ad987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:16:32,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:16:32,572 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:16:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:16:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:16:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f84fa60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:16:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:16:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:32,574 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:16:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a548e18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c4ef34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:16:32,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:16:32,576 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:16:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:16:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:16:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd4cb34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:16:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:16:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:32,577 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:16:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d23b21c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38404fa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,578 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:16:32,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:16:32,579 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:16:32,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:16:32,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:16:32,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2807d0d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:16:32,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:16:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:32,580 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:16:32,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a6deb98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ae0787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:16:32,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:16:32,582 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:16:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:16:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:16:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@583ed8a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:16:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:16:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:32,583 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:16:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b5d2477, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1627c26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:16:32,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:16:32,584 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:16:32,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:16:32,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:16:32,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bfffd6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:16:32,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:16:32,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:32,585 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:16:32,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79bce2b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5993889b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:16:32,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:16:32,587 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:16:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:16:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:16:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@608dfd3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:16:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:16:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:32,588 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:16:32,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c4ecca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117399b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,589 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:16:32,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:16:32,589 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:16:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:16:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:16:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eee537e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:16:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:16:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:32,590 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:16:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fe15b25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:16:32,591 DEBUG [Thread-1882 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:16:32,591 DEBUG [Thread-1884 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:16:32,592 DEBUG [Thread-1873 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:16:32,592 DEBUG [Thread-1877 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:16:32,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:16:32,592 DEBUG [Thread-1875 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:16:32,592 DEBUG [Thread-1886 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:16:32,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:16:32,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:16:32,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:16:32,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:16:32,592 DEBUG [Thread-1871 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:16:32,592 DEBUG [Thread-1880 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:16:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:16:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:16:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:16:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:16:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:16:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:16:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-22T17:16:32,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:16:32,593 DEBUG [Thread-1888 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:16:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:16:32,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:16:32,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:16:32,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:16:32,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:16:32,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:16:32,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:16:32,594 DEBUG [Thread-1880 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:16:32,594 DEBUG [Thread-1869 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:16:32,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:16:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T17:16:32,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:16:32,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:16:32,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:16:32,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:16:32,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:16:32,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:16:32,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:16:32,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:16:32,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:16:32,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:16:32,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:16:32,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:16:32,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:16:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:16:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:16:32,598 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:16:32,598 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:16:32,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:16:32,599 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:16:32,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:16:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:16:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:16:32,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2] 2024-11-22T17:16:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:32,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T17:16:32,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:32,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:32,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:32,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:32,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:32,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:32,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/7537331f0b8548c5af08cb2683d82b50 is 50, key is test_row_0/A:col10/1732295792602/Put/seqid=0 2024-11-22T17:16:32,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742237_1413 (size=16681) 2024-11-22T17:16:32,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/7537331f0b8548c5af08cb2683d82b50 2024-11-22T17:16:32,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/5b162f4646134d1ba3bca2cbc7382c6e is 50, key is test_row_0/B:col10/1732295792602/Put/seqid=0 2024-11-22T17:16:32,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732295792640, max timestamp=1732295792664Num uniques 859; 2024-11-22T17:16:32,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T17:16:32,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732295792640, max timestamp=1732295792664Num uniques 859; 2024-11-22T17:16:32,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T17:16:32,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732295792640, max timestamp=1732295792664Num uniques 859; 2024-11-22T17:16:32,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T17:16:32,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742238_1414 (size=12001) 2024-11-22T17:16:32,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295852684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:32,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295852685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:32,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:32,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:32,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295852686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:32,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295852687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:32,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295852694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T17:16:32,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T17:16:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:32,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:32,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295852801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:32,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:32,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295852801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:32,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295852801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295852802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:32,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295852802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:32,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T17:16:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:32,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:32,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T17:16:33,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295853011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295853011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295853011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295853012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295853012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T17:16:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:33,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:33,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/5b162f4646134d1ba3bca2cbc7382c6e 2024-11-22T17:16:33,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/2a905d4fb8b747a0935829471ae15e04 is 50, key is test_row_0/C:col10/1732295792602/Put/seqid=0 2024-11-22T17:16:33,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742239_1415 (size=12001) 2024-11-22T17:16:33,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T17:16:33,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:33,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:33,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:33,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:33,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:33,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T17:16:33,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295853321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295853331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295853332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295853332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295853332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T17:16:33,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:33,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:33,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:33,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:33,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:33,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/2a905d4fb8b747a0935829471ae15e04 2024-11-22T17:16:33,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/7537331f0b8548c5af08cb2683d82b50 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/7537331f0b8548c5af08cb2683d82b50 2024-11-22T17:16:33,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/7537331f0b8548c5af08cb2683d82b50, entries=250, sequenceid=12, filesize=16.3 K 2024-11-22T17:16:33,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/5b162f4646134d1ba3bca2cbc7382c6e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/5b162f4646134d1ba3bca2cbc7382c6e 2024-11-22T17:16:33,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/5b162f4646134d1ba3bca2cbc7382c6e, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T17:16:33,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/2a905d4fb8b747a0935829471ae15e04 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/2a905d4fb8b747a0935829471ae15e04 2024-11-22T17:16:33,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T17:16:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:33,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:33,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:33,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/2a905d4fb8b747a0935829471ae15e04, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T17:16:33,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 5768c661d3c58ac27b37c9e997559f2f in 917ms, sequenceid=12, compaction requested=false 2024-11-22T17:16:33,521 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T17:16:33,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:33,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T17:16:33,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:33,669 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-22T17:16:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:16:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:16:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:16:33,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/027ee03fd5164f80a71ed54d2b6a178d is 50, key is test_row_0/A:col10/1732295792678/Put/seqid=0 2024-11-22T17:16:33,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742240_1416 (size=12001) 2024-11-22T17:16:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T17:16:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:33,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:33,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295853857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295853858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295853859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295853859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295853862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295853971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295853982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295853982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295853982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295853982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:34,123 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/027ee03fd5164f80a71ed54d2b6a178d 2024-11-22T17:16:34,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/1d897da2ddf244fcad90d1622329a1d2 is 50, key is test_row_0/B:col10/1732295792678/Put/seqid=0 2024-11-22T17:16:34,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742241_1417 (size=12001) 2024-11-22T17:16:34,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295854192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:34,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295854201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295854201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:34,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295854201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:34,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295854201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:34,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:34,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295854501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:34,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295854512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295854512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:34,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:34,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295854521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:34,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295854521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:34,531 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/1d897da2ddf244fcad90d1622329a1d2 2024-11-22T17:16:34,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/fd0eb548e9bc485fb0256a0b4b545f49 is 50, key is test_row_0/C:col10/1732295792678/Put/seqid=0 2024-11-22T17:16:34,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742242_1418 (size=12001) 2024-11-22T17:16:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T17:16:34,941 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/fd0eb548e9bc485fb0256a0b4b545f49 2024-11-22T17:16:34,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/027ee03fd5164f80a71ed54d2b6a178d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/027ee03fd5164f80a71ed54d2b6a178d 2024-11-22T17:16:34,947 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/027ee03fd5164f80a71ed54d2b6a178d, entries=150, sequenceid=45, filesize=11.7 K 2024-11-22T17:16:34,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/1d897da2ddf244fcad90d1622329a1d2 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/1d897da2ddf244fcad90d1622329a1d2 2024-11-22T17:16:34,955 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/1d897da2ddf244fcad90d1622329a1d2, entries=150, sequenceid=45, filesize=11.7 K 2024-11-22T17:16:34,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/fd0eb548e9bc485fb0256a0b4b545f49 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/fd0eb548e9bc485fb0256a0b4b545f49 2024-11-22T17:16:34,959 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/fd0eb548e9bc485fb0256a0b4b545f49, entries=150, sequenceid=45, filesize=11.7 K 2024-11-22T17:16:34,960 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for 5768c661d3c58ac27b37c9e997559f2f in 1291ms, sequenceid=45, compaction requested=false 2024-11-22T17:16:34,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:34,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:34,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-22T17:16:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-22T17:16:34,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-22T17:16:34,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3620 sec 2024-11-22T17:16:34,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.3710 sec 2024-11-22T17:16:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:35,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T17:16:35,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:35,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:35,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:35,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:35,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:35,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:35,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/a9f13562b32d4e5cbc6d145b2f6ff507 is 50, key is test_row_0/A:col10/1732295793860/Put/seqid=0 2024-11-22T17:16:35,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742243_1419 (size=14341) 2024-11-22T17:16:35,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295855067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295855068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295855068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295855068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295855069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295855192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295855192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295855192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295855192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295855192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295855401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295855401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295855401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295855401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295855401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/a9f13562b32d4e5cbc6d145b2f6ff507 2024-11-22T17:16:35,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/fcf5b1e9b67748e185231197c5879cf6 is 50, key is test_row_0/B:col10/1732295793860/Put/seqid=0 2024-11-22T17:16:35,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742244_1420 (size=12001) 2024-11-22T17:16:35,484 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T17:16:35,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295855711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295855711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295855722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295855722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295855722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:35,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/fcf5b1e9b67748e185231197c5879cf6 2024-11-22T17:16:35,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/7f50e66e1fb6499f822285163aa3519b is 50, key is test_row_0/C:col10/1732295793860/Put/seqid=0 2024-11-22T17:16:35,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742245_1421 (size=12001) 2024-11-22T17:16:36,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295856231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:36,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295856231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:36,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295856241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:36,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295856241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:36,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295856241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:36,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/7f50e66e1fb6499f822285163aa3519b 2024-11-22T17:16:36,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/a9f13562b32d4e5cbc6d145b2f6ff507 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a9f13562b32d4e5cbc6d145b2f6ff507 2024-11-22T17:16:36,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a9f13562b32d4e5cbc6d145b2f6ff507, entries=200, sequenceid=57, filesize=14.0 K 2024-11-22T17:16:36,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/fcf5b1e9b67748e185231197c5879cf6 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/fcf5b1e9b67748e185231197c5879cf6 2024-11-22T17:16:36,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/fcf5b1e9b67748e185231197c5879cf6, entries=150, sequenceid=57, filesize=11.7 K 2024-11-22T17:16:36,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/7f50e66e1fb6499f822285163aa3519b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/7f50e66e1fb6499f822285163aa3519b 2024-11-22T17:16:36,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/7f50e66e1fb6499f822285163aa3519b, entries=150, sequenceid=57, filesize=11.7 K 2024-11-22T17:16:36,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5768c661d3c58ac27b37c9e997559f2f in 1246ms, sequenceid=57, compaction requested=true 2024-11-22T17:16:36,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:36,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:16:36,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:36,271 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:36,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:16:36,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:36,271 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:36,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:16:36,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:36,272 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:36,272 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:36,272 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/B is initiating minor compaction (all files) 2024-11-22T17:16:36,272 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/A is initiating minor compaction (all files) 2024-11-22T17:16:36,272 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/B in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:36,272 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/A in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:36,272 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/7537331f0b8548c5af08cb2683d82b50, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/027ee03fd5164f80a71ed54d2b6a178d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a9f13562b32d4e5cbc6d145b2f6ff507] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=42.0 K 2024-11-22T17:16:36,272 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/5b162f4646134d1ba3bca2cbc7382c6e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/1d897da2ddf244fcad90d1622329a1d2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/fcf5b1e9b67748e185231197c5879cf6] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=35.2 K 2024-11-22T17:16:36,273 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b162f4646134d1ba3bca2cbc7382c6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732295792600 2024-11-22T17:16:36,273 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7537331f0b8548c5af08cb2683d82b50, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732295792600 2024-11-22T17:16:36,273 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 027ee03fd5164f80a71ed54d2b6a178d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732295792678 2024-11-22T17:16:36,273 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d897da2ddf244fcad90d1622329a1d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732295792678 2024-11-22T17:16:36,273 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9f13562b32d4e5cbc6d145b2f6ff507, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732295793845 2024-11-22T17:16:36,273 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting fcf5b1e9b67748e185231197c5879cf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732295793845 2024-11-22T17:16:36,279 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#B#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:36,279 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/61e3adeb1e72409296a79bb43904d94d is 50, key is test_row_0/B:col10/1732295793860/Put/seqid=0 2024-11-22T17:16:36,280 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#A#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:36,280 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/10e75d6ca07b4caaadce20a5bd2503ba is 50, key is test_row_0/A:col10/1732295793860/Put/seqid=0 2024-11-22T17:16:36,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742246_1422 (size=12104) 2024-11-22T17:16:36,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742247_1423 (size=12104) 2024-11-22T17:16:36,686 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/61e3adeb1e72409296a79bb43904d94d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/61e3adeb1e72409296a79bb43904d94d 2024-11-22T17:16:36,690 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/B of 5768c661d3c58ac27b37c9e997559f2f into 61e3adeb1e72409296a79bb43904d94d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:36,690 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:36,690 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/B, priority=13, startTime=1732295796271; duration=0sec 2024-11-22T17:16:36,691 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:36,691 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:B 2024-11-22T17:16:36,691 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:36,691 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:36,692 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/C is initiating minor compaction (all files) 2024-11-22T17:16:36,692 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/C in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:36,692 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/2a905d4fb8b747a0935829471ae15e04, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/fd0eb548e9bc485fb0256a0b4b545f49, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/7f50e66e1fb6499f822285163aa3519b] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=35.2 K 2024-11-22T17:16:36,692 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a905d4fb8b747a0935829471ae15e04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732295792600 2024-11-22T17:16:36,692 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting fd0eb548e9bc485fb0256a0b4b545f49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732295792678 2024-11-22T17:16:36,693 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f50e66e1fb6499f822285163aa3519b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732295793845 2024-11-22T17:16:36,700 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/10e75d6ca07b4caaadce20a5bd2503ba as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/10e75d6ca07b4caaadce20a5bd2503ba 2024-11-22T17:16:36,704 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/A of 5768c661d3c58ac27b37c9e997559f2f into 10e75d6ca07b4caaadce20a5bd2503ba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:36,704 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:36,704 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/A, priority=13, startTime=1732295796271; duration=0sec 2024-11-22T17:16:36,704 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:36,704 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:A 2024-11-22T17:16:36,705 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:36,705 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/e453df60edac464c9c9c38673e521292 is 50, key is test_row_0/C:col10/1732295793860/Put/seqid=0 2024-11-22T17:16:36,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742248_1424 (size=12104) 2024-11-22T17:16:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T17:16:36,762 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:36,762 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:16:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-22T17:16:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T17:16:36,764 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:16:36,764 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:16:36,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:16:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T17:16:36,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-22T17:16:36,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:36,916 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T17:16:36,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:36,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:36,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:36,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:36,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:36,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:36,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/31c9b2df806d4c0baca76f2ca487ac30 is 50, key is test_row_0/A:col10/1732295795067/Put/seqid=0 2024-11-22T17:16:36,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742249_1425 (size=12001) 2024-11-22T17:16:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T17:16:37,114 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/e453df60edac464c9c9c38673e521292 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/e453df60edac464c9c9c38673e521292 2024-11-22T17:16:37,118 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/C of 5768c661d3c58ac27b37c9e997559f2f into e453df60edac464c9c9c38673e521292(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:37,119 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:37,119 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/C, priority=13, startTime=1732295796271; duration=0sec 2024-11-22T17:16:37,119 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:37,119 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:C 2024-11-22T17:16:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:37,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:37,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295857262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295857263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,271 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_0' in 5768c661d3c58ac27b37c9e997559f2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295857266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295857269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295857270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,324 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/31c9b2df806d4c0baca76f2ca487ac30 2024-11-22T17:16:37,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/ae0d78d4b4274cdab33203824826e7ab is 50, key is test_row_0/B:col10/1732295795067/Put/seqid=0 2024-11-22T17:16:37,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742250_1426 (size=12001) 2024-11-22T17:16:37,334 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/ae0d78d4b4274cdab33203824826e7ab 2024-11-22T17:16:37,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/96b440349d5f404d957781cb6f9d7ae6 is 50, key is test_row_0/C:col10/1732295795067/Put/seqid=0 2024-11-22T17:16:37,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742251_1427 (size=12001) 2024-11-22T17:16:37,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295857381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295857381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295857381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295857381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T17:16:37,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295857591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295857591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295857591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295857591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,747 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/96b440349d5f404d957781cb6f9d7ae6 2024-11-22T17:16:37,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/31c9b2df806d4c0baca76f2ca487ac30 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/31c9b2df806d4c0baca76f2ca487ac30 2024-11-22T17:16:37,753 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/31c9b2df806d4c0baca76f2ca487ac30, entries=150, sequenceid=79, filesize=11.7 K 2024-11-22T17:16:37,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/ae0d78d4b4274cdab33203824826e7ab as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/ae0d78d4b4274cdab33203824826e7ab 2024-11-22T17:16:37,757 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/ae0d78d4b4274cdab33203824826e7ab, entries=150, sequenceid=79, filesize=11.7 K 2024-11-22T17:16:37,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/96b440349d5f404d957781cb6f9d7ae6 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/96b440349d5f404d957781cb6f9d7ae6 2024-11-22T17:16:37,761 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/96b440349d5f404d957781cb6f9d7ae6, entries=150, sequenceid=79, filesize=11.7 K 2024-11-22T17:16:37,762 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5768c661d3c58ac27b37c9e997559f2f in 846ms, sequenceid=79, compaction requested=false 2024-11-22T17:16:37,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:37,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:37,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-22T17:16:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-22T17:16:37,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-22T17:16:37,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 999 msec 2024-11-22T17:16:37,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.0030 sec 2024-11-22T17:16:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T17:16:37,902 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:37,902 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:16:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-22T17:16:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T17:16:37,903 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:16:37,904 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:16:37,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:16:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:37,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T17:16:37,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:37,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:37,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:37,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:37,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:37,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:37,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/ec0e663dd9ae4dc3a5fd0b46f0cb3cb5 is 50, key is test_row_0/A:col10/1732295797905/Put/seqid=0 2024-11-22T17:16:37,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742252_1428 (size=14341) 2024-11-22T17:16:37,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295857933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295857934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295857946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295857946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T17:16:38,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T17:16:38,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:38,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295858051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:38,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295858051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295858051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295858051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:38,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T17:16:38,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:38,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T17:16:38,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295858271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:38,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295858271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295858271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:38,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295858272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:38,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/ec0e663dd9ae4dc3a5fd0b46f0cb3cb5 2024-11-22T17:16:38,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/8a00b47f51a94ba59f1bff5b17a42e33 is 50, key is test_row_0/B:col10/1732295797905/Put/seqid=0 2024-11-22T17:16:38,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742253_1429 (size=12001) 2024-11-22T17:16:38,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T17:16:38,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:38,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T17:16:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T17:16:38,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:38,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295858581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295858581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:38,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295858591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295858591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:38,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T17:16:38,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:38,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/8a00b47f51a94ba59f1bff5b17a42e33 2024-11-22T17:16:38,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/b69a4c1c260c4800bf6c47cc3b388a2a is 50, key is test_row_0/C:col10/1732295797905/Put/seqid=0 2024-11-22T17:16:38,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742254_1430 (size=12001) 2024-11-22T17:16:38,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T17:16:38,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:38,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T17:16:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:38,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T17:16:39,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295859101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:39,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:39,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295859101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:39,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:39,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:39,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295859101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:39,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:39,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:39,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295859101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:39,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T17:16:39,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:39,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:39,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:39,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:39,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:39,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/b69a4c1c260c4800bf6c47cc3b388a2a 2024-11-22T17:16:39,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/ec0e663dd9ae4dc3a5fd0b46f0cb3cb5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/ec0e663dd9ae4dc3a5fd0b46f0cb3cb5 2024-11-22T17:16:39,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/ec0e663dd9ae4dc3a5fd0b46f0cb3cb5, entries=200, sequenceid=93, filesize=14.0 K 2024-11-22T17:16:39,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/8a00b47f51a94ba59f1bff5b17a42e33 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/8a00b47f51a94ba59f1bff5b17a42e33 2024-11-22T17:16:39,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/8a00b47f51a94ba59f1bff5b17a42e33, entries=150, sequenceid=93, filesize=11.7 K 2024-11-22T17:16:39,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/b69a4c1c260c4800bf6c47cc3b388a2a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/b69a4c1c260c4800bf6c47cc3b388a2a 2024-11-22T17:16:39,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/b69a4c1c260c4800bf6c47cc3b388a2a, entries=150, sequenceid=93, filesize=11.7 K 2024-11-22T17:16:39,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5768c661d3c58ac27b37c9e997559f2f in 1240ms, sequenceid=93, compaction requested=true 2024-11-22T17:16:39,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:39,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:16:39,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:39,146 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:39,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:16:39,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:39,146 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:39,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:16:39,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:39,147 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:39,147 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:39,147 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/B is initiating minor compaction (all files) 2024-11-22T17:16:39,147 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/A is initiating minor compaction (all files) 2024-11-22T17:16:39,147 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/B in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:39,147 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/A in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:39,147 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/61e3adeb1e72409296a79bb43904d94d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/ae0d78d4b4274cdab33203824826e7ab, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/8a00b47f51a94ba59f1bff5b17a42e33] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=35.3 K 2024-11-22T17:16:39,147 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/10e75d6ca07b4caaadce20a5bd2503ba, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/31c9b2df806d4c0baca76f2ca487ac30, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/ec0e663dd9ae4dc3a5fd0b46f0cb3cb5] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=37.5 K 2024-11-22T17:16:39,147 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 10e75d6ca07b4caaadce20a5bd2503ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732295793845 2024-11-22T17:16:39,147 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61e3adeb1e72409296a79bb43904d94d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732295793845 2024-11-22T17:16:39,148 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 31c9b2df806d4c0baca76f2ca487ac30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732295795064 2024-11-22T17:16:39,148 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae0d78d4b4274cdab33203824826e7ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732295795064 2024-11-22T17:16:39,148 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting ec0e663dd9ae4dc3a5fd0b46f0cb3cb5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732295797253 2024-11-22T17:16:39,148 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a00b47f51a94ba59f1bff5b17a42e33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732295797253 2024-11-22T17:16:39,155 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#B#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:39,155 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#A#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:39,155 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/fd250b2891c543c79d588f97d72d16ba is 50, key is test_row_0/A:col10/1732295797905/Put/seqid=0 2024-11-22T17:16:39,156 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/523a7cb4490b453b9a29d12e50ba3c79 is 50, key is test_row_0/B:col10/1732295797905/Put/seqid=0 2024-11-22T17:16:39,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742255_1431 (size=12207) 2024-11-22T17:16:39,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742256_1432 (size=12207) 2024-11-22T17:16:39,164 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/fd250b2891c543c79d588f97d72d16ba as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/fd250b2891c543c79d588f97d72d16ba 2024-11-22T17:16:39,164 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/523a7cb4490b453b9a29d12e50ba3c79 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/523a7cb4490b453b9a29d12e50ba3c79 2024-11-22T17:16:39,167 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/A of 5768c661d3c58ac27b37c9e997559f2f into fd250b2891c543c79d588f97d72d16ba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:39,167 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/B of 5768c661d3c58ac27b37c9e997559f2f into 523a7cb4490b453b9a29d12e50ba3c79(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:39,167 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:39,167 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:39,167 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/B, priority=13, startTime=1732295799146; duration=0sec 2024-11-22T17:16:39,167 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/A, priority=13, startTime=1732295799146; duration=0sec 2024-11-22T17:16:39,167 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:39,167 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:B 2024-11-22T17:16:39,167 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:39,167 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:A 2024-11-22T17:16:39,167 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:39,168 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:39,168 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/C is initiating minor compaction (all files) 2024-11-22T17:16:39,168 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/C in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:39,168 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/e453df60edac464c9c9c38673e521292, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/96b440349d5f404d957781cb6f9d7ae6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/b69a4c1c260c4800bf6c47cc3b388a2a] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=35.3 K 2024-11-22T17:16:39,169 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting e453df60edac464c9c9c38673e521292, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732295793845 2024-11-22T17:16:39,169 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b440349d5f404d957781cb6f9d7ae6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732295795064 2024-11-22T17:16:39,169 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting b69a4c1c260c4800bf6c47cc3b388a2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732295797253 2024-11-22T17:16:39,175 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:39,175 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/3c0418453fa1439ca0bbe781111be85b is 50, key is test_row_0/C:col10/1732295797905/Put/seqid=0 2024-11-22T17:16:39,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742257_1433 (size=12207) 2024-11-22T17:16:39,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T17:16:39,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:39,277 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T17:16:39,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:39,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/f6d45c6618ea4ee3b57797fafa4116c6 is 50, key is test_row_0/A:col10/1732295797932/Put/seqid=0 2024-11-22T17:16:39,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742258_1434 (size=12001) 2024-11-22T17:16:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:39,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:39,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295859353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:39,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:39,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295859461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:39,585 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/3c0418453fa1439ca0bbe781111be85b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/3c0418453fa1439ca0bbe781111be85b 2024-11-22T17:16:39,589 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/C of 5768c661d3c58ac27b37c9e997559f2f into 3c0418453fa1439ca0bbe781111be85b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:39,589 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:39,589 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/C, priority=13, startTime=1732295799146; duration=0sec 2024-11-22T17:16:39,589 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:39,589 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:C 2024-11-22T17:16:39,685 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/f6d45c6618ea4ee3b57797fafa4116c6 2024-11-22T17:16:39,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295859681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:39,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/928150a051984a63bee80d755aa84d29 is 50, key is test_row_0/B:col10/1732295797932/Put/seqid=0 2024-11-22T17:16:39,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742259_1435 (size=12001) 2024-11-22T17:16:39,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295859991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:39,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:39,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T17:16:40,100 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/928150a051984a63bee80d755aa84d29 2024-11-22T17:16:40,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/d307f3cdcc16447881c700443583d19c is 50, key is test_row_0/C:col10/1732295797932/Put/seqid=0 2024-11-22T17:16:40,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742260_1436 (size=12001) 2024-11-22T17:16:40,113 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/d307f3cdcc16447881c700443583d19c 2024-11-22T17:16:40,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/f6d45c6618ea4ee3b57797fafa4116c6 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/f6d45c6618ea4ee3b57797fafa4116c6 2024-11-22T17:16:40,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295860111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:40,118 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 5768c661d3c58ac27b37c9e997559f2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:40,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:40,122 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/f6d45c6618ea4ee3b57797fafa4116c6, entries=150, sequenceid=114, filesize=11.7 K 2024-11-22T17:16:40,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/928150a051984a63bee80d755aa84d29 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/928150a051984a63bee80d755aa84d29 2024-11-22T17:16:40,125 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/928150a051984a63bee80d755aa84d29, entries=150, sequenceid=114, filesize=11.7 K 2024-11-22T17:16:40,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/d307f3cdcc16447881c700443583d19c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d307f3cdcc16447881c700443583d19c 2024-11-22T17:16:40,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295860122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:40,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295860122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:40,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295860122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:40,128 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 5768c661d3c58ac27b37c9e997559f2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:40,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:40,129 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 5768c661d3c58ac27b37c9e997559f2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:40,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:40,129 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_0' in 5768c661d3c58ac27b37c9e997559f2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:40,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:40,132 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d307f3cdcc16447881c700443583d19c, entries=150, sequenceid=114, filesize=11.7 K 2024-11-22T17:16:40,133 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5768c661d3c58ac27b37c9e997559f2f in 855ms, sequenceid=114, compaction requested=false 2024-11-22T17:16:40,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:40,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:40,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-22T17:16:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-22T17:16:40,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-22T17:16:40,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2300 sec 2024-11-22T17:16:40,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.2340 sec 2024-11-22T17:16:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:40,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T17:16:40,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:40,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:40,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:40,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:40,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:40,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:40,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/0fd175a9db9b4a2c8aead5a17ec132f2 is 50, key is test_row_0/A:col10/1732295800506/Put/seqid=0 2024-11-22T17:16:40,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742261_1437 (size=14391) 2024-11-22T17:16:40,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295860616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:40,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295860731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:40,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/0fd175a9db9b4a2c8aead5a17ec132f2 2024-11-22T17:16:40,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/cbb0e03fa3724f61ab4dc189a3947c99 is 50, key is test_row_0/B:col10/1732295800506/Put/seqid=0 2024-11-22T17:16:40,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742262_1438 (size=12051) 2024-11-22T17:16:40,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295860951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:41,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295861261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:41,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/cbb0e03fa3724f61ab4dc189a3947c99 2024-11-22T17:16:41,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/cc1cecb76c634a9bbfef2e08ef61a653 is 50, key is test_row_0/C:col10/1732295800506/Put/seqid=0 2024-11-22T17:16:41,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742263_1439 (size=12051) 2024-11-22T17:16:41,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/cc1cecb76c634a9bbfef2e08ef61a653 2024-11-22T17:16:41,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/0fd175a9db9b4a2c8aead5a17ec132f2 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/0fd175a9db9b4a2c8aead5a17ec132f2 2024-11-22T17:16:41,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/0fd175a9db9b4a2c8aead5a17ec132f2, entries=200, sequenceid=129, filesize=14.1 K 2024-11-22T17:16:41,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/cbb0e03fa3724f61ab4dc189a3947c99 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/cbb0e03fa3724f61ab4dc189a3947c99 2024-11-22T17:16:41,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/cbb0e03fa3724f61ab4dc189a3947c99, entries=150, sequenceid=129, filesize=11.8 K 2024-11-22T17:16:41,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/cc1cecb76c634a9bbfef2e08ef61a653 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/cc1cecb76c634a9bbfef2e08ef61a653 2024-11-22T17:16:41,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/cc1cecb76c634a9bbfef2e08ef61a653, entries=150, sequenceid=129, filesize=11.8 K 2024-11-22T17:16:41,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5768c661d3c58ac27b37c9e997559f2f in 1277ms, sequenceid=129, compaction requested=true 2024-11-22T17:16:41,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:41,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:16:41,786 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:41,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:41,786 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:41,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:16:41,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:41,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:16:41,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:41,787 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:41,787 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:41,787 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/A is initiating minor compaction (all files) 2024-11-22T17:16:41,787 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/B is initiating minor compaction (all files) 2024-11-22T17:16:41,787 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/B in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:41,787 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/A in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:41,787 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/fd250b2891c543c79d588f97d72d16ba, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/f6d45c6618ea4ee3b57797fafa4116c6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/0fd175a9db9b4a2c8aead5a17ec132f2] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=37.7 K 2024-11-22T17:16:41,787 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/523a7cb4490b453b9a29d12e50ba3c79, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/928150a051984a63bee80d755aa84d29, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/cbb0e03fa3724f61ab4dc189a3947c99] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=35.4 K 2024-11-22T17:16:41,788 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 523a7cb4490b453b9a29d12e50ba3c79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732295797253 2024-11-22T17:16:41,788 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd250b2891c543c79d588f97d72d16ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732295797253 2024-11-22T17:16:41,788 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 928150a051984a63bee80d755aa84d29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732295797932 2024-11-22T17:16:41,788 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting cbb0e03fa3724f61ab4dc189a3947c99, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732295799296 2024-11-22T17:16:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:41,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T17:16:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:41,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:41,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:41,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:41,790 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6d45c6618ea4ee3b57797fafa4116c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732295797932 2024-11-22T17:16:41,791 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fd175a9db9b4a2c8aead5a17ec132f2, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732295799296 2024-11-22T17:16:41,795 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#B#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:41,795 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/e2df5b8b9916401e99ed6cdee97950f2 is 50, key is test_row_0/B:col10/1732295800506/Put/seqid=0 2024-11-22T17:16:41,798 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#A#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:41,799 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/bfe501fa864047c884b26bbca2d32baf is 50, key is test_row_0/A:col10/1732295800506/Put/seqid=0 2024-11-22T17:16:41,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/a656f1889b8a4dd5b833dab4610e90aa is 50, key is test_row_0/A:col10/1732295800615/Put/seqid=0 2024-11-22T17:16:41,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742264_1440 (size=12359) 2024-11-22T17:16:41,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742265_1441 (size=12359) 2024-11-22T17:16:41,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742266_1442 (size=12151) 2024-11-22T17:16:41,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295861885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:41,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295861991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T17:16:42,061 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:42,062 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:16:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-22T17:16:42,063 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:16:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T17:16:42,064 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:16:42,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:16:42,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295862132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:42,146 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 5768c661d3c58ac27b37c9e997559f2f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:42,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295862141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:42,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295862142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:42,149 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 5768c661d3c58ac27b37c9e997559f2f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:42,149 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_0' in 5768c661d3c58ac27b37c9e997559f2f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:42,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295862142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:42,151 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 5768c661d3c58ac27b37c9e997559f2f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T17:16:42,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295862202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:42,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-22T17:16:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:42,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:42,236 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/e2df5b8b9916401e99ed6cdee97950f2 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e2df5b8b9916401e99ed6cdee97950f2 2024-11-22T17:16:42,242 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/B of 5768c661d3c58ac27b37c9e997559f2f into e2df5b8b9916401e99ed6cdee97950f2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:42,242 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:42,243 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/B, priority=13, startTime=1732295801786; duration=0sec 2024-11-22T17:16:42,243 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:42,243 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:B 2024-11-22T17:16:42,243 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:42,246 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:42,246 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/C is initiating minor compaction (all files) 2024-11-22T17:16:42,246 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/C in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:42,247 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/3c0418453fa1439ca0bbe781111be85b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d307f3cdcc16447881c700443583d19c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/cc1cecb76c634a9bbfef2e08ef61a653] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=35.4 K 2024-11-22T17:16:42,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/a656f1889b8a4dd5b833dab4610e90aa 2024-11-22T17:16:42,252 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/bfe501fa864047c884b26bbca2d32baf as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bfe501fa864047c884b26bbca2d32baf 2024-11-22T17:16:42,260 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/A of 5768c661d3c58ac27b37c9e997559f2f into bfe501fa864047c884b26bbca2d32baf(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:42,260 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:42,260 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/A, priority=13, startTime=1732295801786; duration=0sec 2024-11-22T17:16:42,260 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:42,261 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:A 2024-11-22T17:16:42,261 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c0418453fa1439ca0bbe781111be85b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732295797253 2024-11-22T17:16:42,263 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting d307f3cdcc16447881c700443583d19c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732295797932 2024-11-22T17:16:42,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/d49f92121d5a4b0bada8c6d635fdf7f4 is 50, key is test_row_0/B:col10/1732295800615/Put/seqid=0 2024-11-22T17:16:42,266 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting cc1cecb76c634a9bbfef2e08ef61a653, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732295799296 2024-11-22T17:16:42,287 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#C#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:42,288 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/d1182b92cccb449db84bb7558d9fd662 is 50, key is test_row_0/C:col10/1732295800506/Put/seqid=0 2024-11-22T17:16:42,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742267_1443 (size=12151) 2024-11-22T17:16:42,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742268_1444 (size=12359) 2024-11-22T17:16:42,339 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/d1182b92cccb449db84bb7558d9fd662 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d1182b92cccb449db84bb7558d9fd662 2024-11-22T17:16:42,349 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/C of 5768c661d3c58ac27b37c9e997559f2f into d1182b92cccb449db84bb7558d9fd662(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:42,349 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:42,349 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/C, priority=13, startTime=1732295801786; duration=0sec 2024-11-22T17:16:42,349 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:42,349 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:C 2024-11-22T17:16:42,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-22T17:16:42,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:42,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:42,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T17:16:42,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295862521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:42,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:42,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:42,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:42,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-22T17:16:42,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:42,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:42,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:42,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:42,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:42,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-22T17:16:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:42,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T17:16:42,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/d49f92121d5a4b0bada8c6d635fdf7f4 2024-11-22T17:16:42,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/027cef4a6bf34effbb146f410bf36d03 is 50, key is test_row_0/C:col10/1732295800615/Put/seqid=0 2024-11-22T17:16:42,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742269_1445 (size=12151) 2024-11-22T17:16:42,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/027cef4a6bf34effbb146f410bf36d03 2024-11-22T17:16:42,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/a656f1889b8a4dd5b833dab4610e90aa as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a656f1889b8a4dd5b833dab4610e90aa 2024-11-22T17:16:42,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a656f1889b8a4dd5b833dab4610e90aa, entries=150, sequenceid=148, filesize=11.9 K 2024-11-22T17:16:42,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/d49f92121d5a4b0bada8c6d635fdf7f4 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/d49f92121d5a4b0bada8c6d635fdf7f4 2024-11-22T17:16:42,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/d49f92121d5a4b0bada8c6d635fdf7f4, entries=150, sequenceid=148, filesize=11.9 K 2024-11-22T17:16:42,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/027cef4a6bf34effbb146f410bf36d03 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/027cef4a6bf34effbb146f410bf36d03 2024-11-22T17:16:42,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/027cef4a6bf34effbb146f410bf36d03, entries=150, sequenceid=148, filesize=11.9 K 2024-11-22T17:16:42,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5768c661d3c58ac27b37c9e997559f2f in 965ms, sequenceid=148, compaction requested=false 2024-11-22T17:16:42,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:42,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-22T17:16:42,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:42,837 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T17:16:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:42,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/aaf1b2f9473a4a6db95c52162aa83349 is 50, key is test_row_0/A:col10/1732295801854/Put/seqid=0 2024-11-22T17:16:42,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742270_1446 (size=12151) 2024-11-22T17:16:42,893 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/aaf1b2f9473a4a6db95c52162aa83349 2024-11-22T17:16:42,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/c711ab2f050740c292d92900667a4a39 is 50, key is test_row_0/B:col10/1732295801854/Put/seqid=0 2024-11-22T17:16:42,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742271_1447 (size=12151) 2024-11-22T17:16:42,985 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/c711ab2f050740c292d92900667a4a39 2024-11-22T17:16:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/a65745f04854406a972d5df32619881e is 50, key is test_row_0/C:col10/1732295801854/Put/seqid=0 2024-11-22T17:16:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:43,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:43,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742272_1448 (size=12151) 2024-11-22T17:16:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T17:16:43,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295863214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:43,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295863331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:43,458 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/a65745f04854406a972d5df32619881e 2024-11-22T17:16:43,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/aaf1b2f9473a4a6db95c52162aa83349 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/aaf1b2f9473a4a6db95c52162aa83349 2024-11-22T17:16:43,475 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/aaf1b2f9473a4a6db95c52162aa83349, entries=150, sequenceid=164, filesize=11.9 K 2024-11-22T17:16:43,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/c711ab2f050740c292d92900667a4a39 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/c711ab2f050740c292d92900667a4a39 2024-11-22T17:16:43,482 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/c711ab2f050740c292d92900667a4a39, entries=150, sequenceid=164, filesize=11.9 K 2024-11-22T17:16:43,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/a65745f04854406a972d5df32619881e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/a65745f04854406a972d5df32619881e 2024-11-22T17:16:43,487 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/a65745f04854406a972d5df32619881e, entries=150, sequenceid=164, filesize=11.9 K 2024-11-22T17:16:43,490 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5768c661d3c58ac27b37c9e997559f2f in 653ms, sequenceid=164, compaction requested=true 2024-11-22T17:16:43,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:43,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:43,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-22T17:16:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-22T17:16:43,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-22T17:16:43,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4310 sec 2024-11-22T17:16:43,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4350 sec 2024-11-22T17:16:43,516 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T17:16:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:43,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T17:16:43,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:43,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:43,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:43,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:43,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:43,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:43,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/db23c172315148efaadc14e48fdd9fca is 50, key is test_row_0/A:col10/1732295803546/Put/seqid=0 2024-11-22T17:16:43,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742273_1449 (size=14541) 2024-11-22T17:16:43,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295863640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:43,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295863761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:43,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:43,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:43,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:43,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295863981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:44,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/db23c172315148efaadc14e48fdd9fca 2024-11-22T17:16:44,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/0c9189a2576647d7bdd55e55f9a868d5 is 50, key is test_row_0/B:col10/1732295803546/Put/seqid=0 2024-11-22T17:16:44,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742274_1450 (size=12151) 2024-11-22T17:16:44,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/0c9189a2576647d7bdd55e55f9a868d5 2024-11-22T17:16:44,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/488961a06acd41839bc71002b8a3a3e5 is 50, key is test_row_0/C:col10/1732295803546/Put/seqid=0 2024-11-22T17:16:44,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742275_1451 (size=12151) 2024-11-22T17:16:44,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/488961a06acd41839bc71002b8a3a3e5 2024-11-22T17:16:44,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/db23c172315148efaadc14e48fdd9fca as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/db23c172315148efaadc14e48fdd9fca 2024-11-22T17:16:44,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/db23c172315148efaadc14e48fdd9fca, entries=200, sequenceid=184, filesize=14.2 K 2024-11-22T17:16:44,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/0c9189a2576647d7bdd55e55f9a868d5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/0c9189a2576647d7bdd55e55f9a868d5 2024-11-22T17:16:44,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/0c9189a2576647d7bdd55e55f9a868d5, entries=150, sequenceid=184, filesize=11.9 K 2024-11-22T17:16:44,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/488961a06acd41839bc71002b8a3a3e5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/488961a06acd41839bc71002b8a3a3e5 2024-11-22T17:16:44,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/488961a06acd41839bc71002b8a3a3e5, entries=150, sequenceid=184, filesize=11.9 K 2024-11-22T17:16:44,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5768c661d3c58ac27b37c9e997559f2f in 559ms, sequenceid=184, compaction requested=true 2024-11-22T17:16:44,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:44,108 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:16:44,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:16:44,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:44,109 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:16:44,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:16:44,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:44,109 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51202 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:16:44,109 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/A is initiating minor compaction (all files) 2024-11-22T17:16:44,110 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/A in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:44,110 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bfe501fa864047c884b26bbca2d32baf, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a656f1889b8a4dd5b833dab4610e90aa, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/aaf1b2f9473a4a6db95c52162aa83349, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/db23c172315148efaadc14e48fdd9fca] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=50.0 K 2024-11-22T17:16:44,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:16:44,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:44,110 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfe501fa864047c884b26bbca2d32baf, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732295799296 2024-11-22T17:16:44,110 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:16:44,110 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/B is initiating minor compaction (all files) 2024-11-22T17:16:44,110 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/B in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:44,110 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e2df5b8b9916401e99ed6cdee97950f2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/d49f92121d5a4b0bada8c6d635fdf7f4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/c711ab2f050740c292d92900667a4a39, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/0c9189a2576647d7bdd55e55f9a868d5] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=47.7 K 2024-11-22T17:16:44,110 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting a656f1889b8a4dd5b833dab4610e90aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732295800592 2024-11-22T17:16:44,111 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting e2df5b8b9916401e99ed6cdee97950f2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732295799296 2024-11-22T17:16:44,111 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaf1b2f9473a4a6db95c52162aa83349, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732295801854 2024-11-22T17:16:44,111 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting d49f92121d5a4b0bada8c6d635fdf7f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732295800592 2024-11-22T17:16:44,111 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting db23c172315148efaadc14e48fdd9fca, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732295803184 2024-11-22T17:16:44,111 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting c711ab2f050740c292d92900667a4a39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732295801854 2024-11-22T17:16:44,111 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c9189a2576647d7bdd55e55f9a868d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732295803184 2024-11-22T17:16:44,123 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#B#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:44,124 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#A#compaction#379 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:44,124 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/1e47925ad4594e519e2455d4e720ed9e is 50, key is test_row_0/B:col10/1732295803546/Put/seqid=0 2024-11-22T17:16:44,124 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/bcf05b06b96341a59454376bdbfc98bb is 50, key is test_row_0/A:col10/1732295803546/Put/seqid=0 2024-11-22T17:16:44,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742277_1453 (size=12595) 2024-11-22T17:16:44,150 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/bcf05b06b96341a59454376bdbfc98bb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bcf05b06b96341a59454376bdbfc98bb 2024-11-22T17:16:44,155 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/A of 5768c661d3c58ac27b37c9e997559f2f into bcf05b06b96341a59454376bdbfc98bb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:44,155 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:44,155 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/A, priority=12, startTime=1732295804108; duration=0sec 2024-11-22T17:16:44,155 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:44,155 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:A 2024-11-22T17:16:44,155 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:16:44,156 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:16:44,156 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/C is initiating minor compaction (all files) 2024-11-22T17:16:44,156 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/C in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:44,156 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d1182b92cccb449db84bb7558d9fd662, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/027cef4a6bf34effbb146f410bf36d03, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/a65745f04854406a972d5df32619881e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/488961a06acd41839bc71002b8a3a3e5] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=47.7 K 2024-11-22T17:16:44,157 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1182b92cccb449db84bb7558d9fd662, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732295799296 2024-11-22T17:16:44,157 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 027cef4a6bf34effbb146f410bf36d03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732295800592 2024-11-22T17:16:44,158 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting a65745f04854406a972d5df32619881e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732295801854 2024-11-22T17:16:44,158 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 488961a06acd41839bc71002b8a3a3e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732295803184 2024-11-22T17:16:44,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742276_1452 (size=12595) 2024-11-22T17:16:44,168 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/1e47925ad4594e519e2455d4e720ed9e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/1e47925ad4594e519e2455d4e720ed9e 2024-11-22T17:16:44,173 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:44,174 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/0cce11c53d5a44cfb6a3be4f1dfdb673 is 50, key is test_row_0/C:col10/1732295803546/Put/seqid=0 2024-11-22T17:16:44,178 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/B of 5768c661d3c58ac27b37c9e997559f2f into 1e47925ad4594e519e2455d4e720ed9e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:44,178 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:44,178 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/B, priority=12, startTime=1732295804109; duration=0sec 2024-11-22T17:16:44,178 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:44,178 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:B 2024-11-22T17:16:44,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742278_1454 (size=12595) 2024-11-22T17:16:44,192 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/0cce11c53d5a44cfb6a3be4f1dfdb673 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/0cce11c53d5a44cfb6a3be4f1dfdb673 2024-11-22T17:16:44,197 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/C of 5768c661d3c58ac27b37c9e997559f2f into 0cce11c53d5a44cfb6a3be4f1dfdb673(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:44,197 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:44,197 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/C, priority=12, startTime=1732295804109; duration=0sec 2024-11-22T17:16:44,200 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:44,201 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:C 2024-11-22T17:16:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T17:16:44,212 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:44,212 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:16:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-22T17:16:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T17:16:44,218 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:16:44,218 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:16:44,218 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:16:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:44,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T17:16:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:44,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/af585ddf45614a169eab215fe6774638 is 50, key is test_row_0/A:col10/1732295804304/Put/seqid=0 2024-11-22T17:16:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T17:16:44,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742279_1455 (size=14541) 2024-11-22T17:16:44,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:44,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:44,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:44,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:44,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295864435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:44,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:44,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:44,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:44,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:44,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T17:16:44,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295864551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:44,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:44,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:44,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:44,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:44,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/af585ddf45614a169eab215fe6774638 2024-11-22T17:16:44,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/e768c1271ae645458e599089a562c562 is 50, key is test_row_0/B:col10/1732295804304/Put/seqid=0 2024-11-22T17:16:44,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742280_1456 (size=12151) 2024-11-22T17:16:44,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295864761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:44,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:44,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:44,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:44,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:44,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T17:16:44,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:44,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:44,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:44,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:44,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295865081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:45,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:45,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:45,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:45,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:45,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/e768c1271ae645458e599089a562c562 2024-11-22T17:16:45,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/a6cc1bad0478467abcdd11ddc3cfece1 is 50, key is test_row_0/C:col10/1732295804304/Put/seqid=0 2024-11-22T17:16:45,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742281_1457 (size=12151) 2024-11-22T17:16:45,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:45,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:45,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:45,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:45,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T17:16:45,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:45,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:45,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:45,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:45,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/a6cc1bad0478467abcdd11ddc3cfece1 2024-11-22T17:16:45,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/af585ddf45614a169eab215fe6774638 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/af585ddf45614a169eab215fe6774638 2024-11-22T17:16:45,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/af585ddf45614a169eab215fe6774638, entries=200, sequenceid=200, filesize=14.2 K 2024-11-22T17:16:45,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/e768c1271ae645458e599089a562c562 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e768c1271ae645458e599089a562c562 2024-11-22T17:16:45,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e768c1271ae645458e599089a562c562, entries=150, sequenceid=200, filesize=11.9 K 2024-11-22T17:16:45,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:45,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/a6cc1bad0478467abcdd11ddc3cfece1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/a6cc1bad0478467abcdd11ddc3cfece1 2024-11-22T17:16:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/a6cc1bad0478467abcdd11ddc3cfece1, entries=150, sequenceid=200, filesize=11.9 K 2024-11-22T17:16:45,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5768c661d3c58ac27b37c9e997559f2f in 1300ms, sequenceid=200, compaction requested=false 2024-11-22T17:16:45,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:45,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T17:16:45,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:45,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:45,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:45,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:45,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:45,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:45,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/b0ab7b96c2f540048b2fd4e7068d0b4b is 50, key is test_row_0/A:col10/1732295805607/Put/seqid=0 2024-11-22T17:16:45,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742282_1458 (size=14541) 2024-11-22T17:16:45,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295865684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:45,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:45,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:45,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:45,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:45,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295865801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:45,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:45,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:46,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295866011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:46,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:46,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/b0ab7b96c2f540048b2fd4e7068d0b4b 2024-11-22T17:16:46,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/e53eb24472f34d99a89e99e937583d14 is 50, key is test_row_0/B:col10/1732295805607/Put/seqid=0 2024-11-22T17:16:46,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742283_1459 (size=12151) 2024-11-22T17:16:46,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:46,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:46,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:46,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:46,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:46,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48152 deadline: 1732295866161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:46,170 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 5768c661d3c58ac27b37c9e997559f2f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:46,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48222 deadline: 1732295866182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:46,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1732295866182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:46,187 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 5768c661d3c58ac27b37c9e997559f2f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:46,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:46,187 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_0' in 5768c661d3c58ac27b37c9e997559f2f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:46,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:46,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48180 deadline: 1732295866191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:46,199 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 5768c661d3c58ac27b37c9e997559f2f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:16:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:46,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:46,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:46,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:46,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:46,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:46,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:46,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295866321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T17:16:46,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:46,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:46,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:46,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:46,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:46,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:46,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/e53eb24472f34d99a89e99e937583d14 2024-11-22T17:16:46,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/da3fbaa786ec4692aa579ebeb0c173a4 is 50, key is test_row_0/C:col10/1732295805607/Put/seqid=0 2024-11-22T17:16:46,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742284_1460 (size=12151) 2024-11-22T17:16:46,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/da3fbaa786ec4692aa579ebeb0c173a4 2024-11-22T17:16:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/b0ab7b96c2f540048b2fd4e7068d0b4b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/b0ab7b96c2f540048b2fd4e7068d0b4b 2024-11-22T17:16:46,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/b0ab7b96c2f540048b2fd4e7068d0b4b, entries=200, sequenceid=220, filesize=14.2 K 2024-11-22T17:16:46,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/e53eb24472f34d99a89e99e937583d14 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e53eb24472f34d99a89e99e937583d14 2024-11-22T17:16:46,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e53eb24472f34d99a89e99e937583d14, entries=150, sequenceid=220, filesize=11.9 K 2024-11-22T17:16:46,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/da3fbaa786ec4692aa579ebeb0c173a4 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/da3fbaa786ec4692aa579ebeb0c173a4 2024-11-22T17:16:46,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/da3fbaa786ec4692aa579ebeb0c173a4, entries=150, sequenceid=220, filesize=11.9 K 2024-11-22T17:16:46,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5768c661d3c58ac27b37c9e997559f2f in 906ms, sequenceid=220, compaction requested=true 2024-11-22T17:16:46,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:46,516 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:46,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:16:46,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:46,516 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:46,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T17:16:46,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:46,518 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T17:16:46,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:46,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:46,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:46,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:46,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:46,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:46,522 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:46,522 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/A is initiating minor compaction (all files) 2024-11-22T17:16:46,522 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/A in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:46,522 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bcf05b06b96341a59454376bdbfc98bb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/af585ddf45614a169eab215fe6774638, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/b0ab7b96c2f540048b2fd4e7068d0b4b] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=40.7 K 2024-11-22T17:16:46,523 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:46,523 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/B is initiating minor compaction (all files) 2024-11-22T17:16:46,523 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/B in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:46,523 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/1e47925ad4594e519e2455d4e720ed9e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e768c1271ae645458e599089a562c562, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e53eb24472f34d99a89e99e937583d14] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=36.0 K 2024-11-22T17:16:46,524 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcf05b06b96341a59454376bdbfc98bb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732295803184 2024-11-22T17:16:46,524 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e47925ad4594e519e2455d4e720ed9e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732295803184 2024-11-22T17:16:46,525 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting af585ddf45614a169eab215fe6774638, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732295803582 2024-11-22T17:16:46,526 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting e768c1271ae645458e599089a562c562, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732295803582 2024-11-22T17:16:46,526 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0ab7b96c2f540048b2fd4e7068d0b4b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732295804424 2024-11-22T17:16:46,526 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting e53eb24472f34d99a89e99e937583d14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732295804424 2024-11-22T17:16:46,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:16:46,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:46,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:16:46,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:46,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/91bc18f880e44281aabb49536f6e85e5 is 50, key is test_row_0/A:col10/1732295805636/Put/seqid=0 2024-11-22T17:16:46,544 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#A#compaction#388 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:46,544 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/f1276ac9b75a4e26a87f437c457cab31 is 50, key is test_row_0/A:col10/1732295805607/Put/seqid=0 2024-11-22T17:16:46,546 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#B#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:46,546 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/9113b2f8bf284478b44313a55fe907ae is 50, key is test_row_0/B:col10/1732295805607/Put/seqid=0 2024-11-22T17:16:46,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742285_1461 (size=12151) 2024-11-22T17:16:46,576 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/91bc18f880e44281aabb49536f6e85e5 2024-11-22T17:16:46,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742286_1462 (size=12697) 2024-11-22T17:16:46,582 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/9113b2f8bf284478b44313a55fe907ae as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/9113b2f8bf284478b44313a55fe907ae 2024-11-22T17:16:46,588 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/B of 5768c661d3c58ac27b37c9e997559f2f into 9113b2f8bf284478b44313a55fe907ae(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:46,588 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:46,588 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/B, priority=13, startTime=1732295806516; duration=0sec 2024-11-22T17:16:46,588 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:46,588 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:B 2024-11-22T17:16:46,588 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:46,591 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:46,591 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/C is initiating minor compaction (all files) 2024-11-22T17:16:46,592 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/C in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:46,592 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/0cce11c53d5a44cfb6a3be4f1dfdb673, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/a6cc1bad0478467abcdd11ddc3cfece1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/da3fbaa786ec4692aa579ebeb0c173a4] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=36.0 K 2024-11-22T17:16:46,592 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cce11c53d5a44cfb6a3be4f1dfdb673, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732295803184 2024-11-22T17:16:46,596 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a6cc1bad0478467abcdd11ddc3cfece1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732295803582 2024-11-22T17:16:46,596 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting da3fbaa786ec4692aa579ebeb0c173a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732295804424 2024-11-22T17:16:46,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/11709cf1b1844f3e852bcc30467afca6 is 50, key is test_row_0/B:col10/1732295805636/Put/seqid=0 2024-11-22T17:16:46,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742287_1463 (size=12697) 2024-11-22T17:16:46,616 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/f1276ac9b75a4e26a87f437c457cab31 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/f1276ac9b75a4e26a87f437c457cab31 2024-11-22T17:16:46,625 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/A of 5768c661d3c58ac27b37c9e997559f2f into f1276ac9b75a4e26a87f437c457cab31(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:46,625 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:46,625 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/A, priority=13, startTime=1732295806516; duration=0sec 2024-11-22T17:16:46,625 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:46,625 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:A 2024-11-22T17:16:46,627 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#C#compaction#391 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:46,629 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/9fa8585bc60c4606855782a556ba8b4d is 50, key is test_row_0/C:col10/1732295805607/Put/seqid=0 2024-11-22T17:16:46,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742288_1464 (size=12151) 2024-11-22T17:16:46,632 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/11709cf1b1844f3e852bcc30467afca6 2024-11-22T17:16:46,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/576c4c46d44940059ac50728bb98e836 is 50, key is test_row_0/C:col10/1732295805636/Put/seqid=0 2024-11-22T17:16:46,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742289_1465 (size=12697) 2024-11-22T17:16:46,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742290_1466 (size=12151) 2024-11-22T17:16:46,687 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/9fa8585bc60c4606855782a556ba8b4d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/9fa8585bc60c4606855782a556ba8b4d 2024-11-22T17:16:46,693 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/C of 5768c661d3c58ac27b37c9e997559f2f into 9fa8585bc60c4606855782a556ba8b4d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:46,693 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:46,693 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/C, priority=13, startTime=1732295806533; duration=0sec 2024-11-22T17:16:46,693 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:46,693 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:C 2024-11-22T17:16:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:46,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:46,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295866932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:47,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:47,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295867041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:47,119 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/576c4c46d44940059ac50728bb98e836 2024-11-22T17:16:47,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/91bc18f880e44281aabb49536f6e85e5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/91bc18f880e44281aabb49536f6e85e5 2024-11-22T17:16:47,125 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/91bc18f880e44281aabb49536f6e85e5, entries=150, sequenceid=232, filesize=11.9 K 2024-11-22T17:16:47,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/11709cf1b1844f3e852bcc30467afca6 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/11709cf1b1844f3e852bcc30467afca6 2024-11-22T17:16:47,130 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/11709cf1b1844f3e852bcc30467afca6, entries=150, sequenceid=232, filesize=11.9 K 2024-11-22T17:16:47,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/576c4c46d44940059ac50728bb98e836 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/576c4c46d44940059ac50728bb98e836 2024-11-22T17:16:47,134 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/576c4c46d44940059ac50728bb98e836, entries=150, sequenceid=232, filesize=11.9 K 2024-11-22T17:16:47,135 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5768c661d3c58ac27b37c9e997559f2f in 617ms, sequenceid=232, compaction requested=false 2024-11-22T17:16:47,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:47,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:47,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-22T17:16:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-22T17:16:47,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-22T17:16:47,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9180 sec 2024-11-22T17:16:47,139 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.9260 sec 2024-11-22T17:16:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:47,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-22T17:16:47,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:47,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:16:47,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:47,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:16:47,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:47,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:16:47,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/bc6c27c138ff41bd822b963b8c700a8b is 50, key is test_row_0/A:col10/1732295807254/Put/seqid=0 2024-11-22T17:16:47,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742291_1467 (size=14541) 2024-11-22T17:16:47,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295867308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:47,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295867421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:47,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295867631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:47,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/bc6c27c138ff41bd822b963b8c700a8b 2024-11-22T17:16:47,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/b720947529814553a1782d5e5b64403c is 50, key is test_row_0/B:col10/1732295807254/Put/seqid=0 2024-11-22T17:16:47,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742292_1468 (size=12151) 2024-11-22T17:16:47,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295867951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:47,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:47,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:47,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:48,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/b720947529814553a1782d5e5b64403c 2024-11-22T17:16:48,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/1f8fe8f4ee884c8d99da3f653672475b is 50, key is test_row_0/C:col10/1732295807254/Put/seqid=0 2024-11-22T17:16:48,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742293_1469 (size=12151) 2024-11-22T17:16:48,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/1f8fe8f4ee884c8d99da3f653672475b 2024-11-22T17:16:48,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/bc6c27c138ff41bd822b963b8c700a8b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bc6c27c138ff41bd822b963b8c700a8b 2024-11-22T17:16:48,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bc6c27c138ff41bd822b963b8c700a8b, entries=200, sequenceid=256, filesize=14.2 K 2024-11-22T17:16:48,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/b720947529814553a1782d5e5b64403c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/b720947529814553a1782d5e5b64403c 2024-11-22T17:16:48,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/b720947529814553a1782d5e5b64403c, entries=150, sequenceid=256, filesize=11.9 K 2024-11-22T17:16:48,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/1f8fe8f4ee884c8d99da3f653672475b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/1f8fe8f4ee884c8d99da3f653672475b 2024-11-22T17:16:48,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/1f8fe8f4ee884c8d99da3f653672475b, entries=150, sequenceid=256, filesize=11.9 K 2024-11-22T17:16:48,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 5768c661d3c58ac27b37c9e997559f2f in 888ms, sequenceid=256, compaction requested=true 2024-11-22T17:16:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:48,145 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:16:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:48,145 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:16:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:16:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:48,151 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:48,151 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39389 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:48,151 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/A is initiating minor compaction (all files) 2024-11-22T17:16:48,151 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/B is initiating minor compaction (all files) 2024-11-22T17:16:48,151 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/A in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:48,152 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/f1276ac9b75a4e26a87f437c457cab31, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/91bc18f880e44281aabb49536f6e85e5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bc6c27c138ff41bd822b963b8c700a8b] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=38.5 K 2024-11-22T17:16:48,152 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/B in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:48,152 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/9113b2f8bf284478b44313a55fe907ae, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/11709cf1b1844f3e852bcc30467afca6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/b720947529814553a1782d5e5b64403c] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=36.1 K 2024-11-22T17:16:48,155 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1276ac9b75a4e26a87f437c457cab31, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732295804424 2024-11-22T17:16:48,160 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 9113b2f8bf284478b44313a55fe907ae, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732295804424 2024-11-22T17:16:48,161 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 11709cf1b1844f3e852bcc30467afca6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732295805629 2024-11-22T17:16:48,161 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting b720947529814553a1782d5e5b64403c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732295806909 2024-11-22T17:16:48,169 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91bc18f880e44281aabb49536f6e85e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732295805629 2024-11-22T17:16:48,170 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#B#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:48,170 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/f16a9560ac9e4de5a51a098b5021e03d is 50, key is test_row_0/B:col10/1732295807254/Put/seqid=0 2024-11-22T17:16:48,171 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc6c27c138ff41bd822b963b8c700a8b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732295806909 2024-11-22T17:16:48,179 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:48,180 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/a6628c0bb3194ae2a69b5703035261bb is 50, key is test_row_0/A:col10/1732295807254/Put/seqid=0 2024-11-22T17:16:48,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742294_1470 (size=12799) 2024-11-22T17:16:48,199 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/f16a9560ac9e4de5a51a098b5021e03d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/f16a9560ac9e4de5a51a098b5021e03d 2024-11-22T17:16:48,208 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/B of 5768c661d3c58ac27b37c9e997559f2f into f16a9560ac9e4de5a51a098b5021e03d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:48,208 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:48,208 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/B, priority=13, startTime=1732295808145; duration=0sec 2024-11-22T17:16:48,208 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:48,208 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:B 2024-11-22T17:16:48,208 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:16:48,212 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:16:48,212 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/C is initiating minor compaction (all files) 2024-11-22T17:16:48,212 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/C in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:48,213 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/9fa8585bc60c4606855782a556ba8b4d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/576c4c46d44940059ac50728bb98e836, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/1f8fe8f4ee884c8d99da3f653672475b] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=36.1 K 2024-11-22T17:16:48,213 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fa8585bc60c4606855782a556ba8b4d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732295804424 2024-11-22T17:16:48,214 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 576c4c46d44940059ac50728bb98e836, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732295805629 2024-11-22T17:16:48,214 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f8fe8f4ee884c8d99da3f653672475b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732295806909 2024-11-22T17:16:48,222 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#C#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:48,222 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/61a3991f66a14540b04c7b9f99e07be2 is 50, key is test_row_0/C:col10/1732295807254/Put/seqid=0 2024-11-22T17:16:48,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742295_1471 (size=12799) 2024-11-22T17:16:48,249 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/a6628c0bb3194ae2a69b5703035261bb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a6628c0bb3194ae2a69b5703035261bb 2024-11-22T17:16:48,266 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/A of 5768c661d3c58ac27b37c9e997559f2f into a6628c0bb3194ae2a69b5703035261bb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:48,267 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:48,267 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/A, priority=13, startTime=1732295808145; duration=0sec 2024-11-22T17:16:48,267 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:48,267 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:A 2024-11-22T17:16:48,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742296_1472 (size=12799) 2024-11-22T17:16:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T17:16:48,391 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:48,392 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:16:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-22T17:16:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T17:16:48,393 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:16:48,394 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:16:48,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:16:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:48,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T17:16:48,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:48,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:48,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:48,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:48,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:48,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:48,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/d24ed07446f04781b2ceb5dd04a0e130 is 50, key is test_row_0/A:col10/1732295808465/Put/seqid=0 2024-11-22T17:16:48,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742297_1473 (size=14741) 2024-11-22T17:16:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T17:16:48,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-22T17:16:48,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:48,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:48,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:48,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:48,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295868571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:48,677 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/61a3991f66a14540b04c7b9f99e07be2 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/61a3991f66a14540b04c7b9f99e07be2 2024-11-22T17:16:48,681 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/C of 5768c661d3c58ac27b37c9e997559f2f into 61a3991f66a14540b04c7b9f99e07be2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:48,681 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:48,681 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/C, priority=13, startTime=1732295808145; duration=0sec 2024-11-22T17:16:48,681 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:48,681 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:C 2024-11-22T17:16:48,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295868681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:48,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-22T17:16:48,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:48,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:48,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:48,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:48,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T17:16:48,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-22T17:16:48,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:48,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:48,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/d24ed07446f04781b2ceb5dd04a0e130 2024-11-22T17:16:48,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/2667ad9773c54976be7c45d968168c7c is 50, key is test_row_0/B:col10/1732295808465/Put/seqid=0 2024-11-22T17:16:48,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:48,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295868891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:48,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742298_1474 (size=12301) 2024-11-22T17:16:49,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-22T17:16:49,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:49,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:49,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:49,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T17:16:49,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-22T17:16:49,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:49,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:49,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295869211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:49,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/2667ad9773c54976be7c45d968168c7c 2024-11-22T17:16:49,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-22T17:16:49,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:49,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:49,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:49,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/c80f2b1280c547779b6e5152fdcae7b1 is 50, key is test_row_0/C:col10/1732295808465/Put/seqid=0 2024-11-22T17:16:49,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742299_1475 (size=12301) 2024-11-22T17:16:49,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-22T17:16:49,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:49,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:49,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:49,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T17:16:49,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-22T17:16:49,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:49,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:49,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:49,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:49,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/c80f2b1280c547779b6e5152fdcae7b1 2024-11-22T17:16:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/d24ed07446f04781b2ceb5dd04a0e130 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/d24ed07446f04781b2ceb5dd04a0e130 2024-11-22T17:16:49,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295869731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:49,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/d24ed07446f04781b2ceb5dd04a0e130, entries=200, sequenceid=270, filesize=14.4 K 2024-11-22T17:16:49,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/2667ad9773c54976be7c45d968168c7c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/2667ad9773c54976be7c45d968168c7c 2024-11-22T17:16:49,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/2667ad9773c54976be7c45d968168c7c, entries=150, sequenceid=270, filesize=12.0 K 2024-11-22T17:16:49,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/c80f2b1280c547779b6e5152fdcae7b1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/c80f2b1280c547779b6e5152fdcae7b1 2024-11-22T17:16:49,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/c80f2b1280c547779b6e5152fdcae7b1, entries=150, sequenceid=270, filesize=12.0 K 2024-11-22T17:16:49,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5768c661d3c58ac27b37c9e997559f2f in 1285ms, sequenceid=270, compaction requested=false 2024-11-22T17:16:49,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:49,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-22T17:16:49,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:49,773 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T17:16:49,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:49,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:49,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:49,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:49,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:49,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:49,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/d09d9ba111504b8396a084fbb4fe70d9 is 50, key is test_row_0/A:col10/1732295808570/Put/seqid=0 2024-11-22T17:16:49,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742300_1476 (size=12301) 2024-11-22T17:16:49,797 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/d09d9ba111504b8396a084fbb4fe70d9 2024-11-22T17:16:49,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/d4c979874923495f9d70e648165e2b0c is 50, key is test_row_0/B:col10/1732295808570/Put/seqid=0 2024-11-22T17:16:49,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742301_1477 (size=12301) 2024-11-22T17:16:50,216 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/d4c979874923495f9d70e648165e2b0c 2024-11-22T17:16:50,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/6207817291034fd0858963bf999f4125 is 50, key is test_row_0/C:col10/1732295808570/Put/seqid=0 2024-11-22T17:16:50,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742302_1478 (size=12301) 2024-11-22T17:16:50,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T17:16:50,625 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/6207817291034fd0858963bf999f4125 2024-11-22T17:16:50,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/d09d9ba111504b8396a084fbb4fe70d9 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/d09d9ba111504b8396a084fbb4fe70d9 2024-11-22T17:16:50,630 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/d09d9ba111504b8396a084fbb4fe70d9, entries=150, sequenceid=291, filesize=12.0 K 2024-11-22T17:16:50,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/d4c979874923495f9d70e648165e2b0c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/d4c979874923495f9d70e648165e2b0c 2024-11-22T17:16:50,634 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/d4c979874923495f9d70e648165e2b0c, entries=150, sequenceid=291, filesize=12.0 K 2024-11-22T17:16:50,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/6207817291034fd0858963bf999f4125 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/6207817291034fd0858963bf999f4125 2024-11-22T17:16:50,637 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/6207817291034fd0858963bf999f4125, entries=150, sequenceid=291, filesize=12.0 K 2024-11-22T17:16:50,638 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 5768c661d3c58ac27b37c9e997559f2f in 865ms, sequenceid=291, compaction requested=true 2024-11-22T17:16:50,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:50,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:50,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-22T17:16:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-22T17:16:50,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-22T17:16:50,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2450 sec 2024-11-22T17:16:50,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.2490 sec 2024-11-22T17:16:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:50,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:16:50,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:50,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:50,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:50,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:50,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:50,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:50,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/92cff8478a4f4b00b3cb79dca278cbad is 50, key is test_row_0/A:col10/1732295810767/Put/seqid=0 2024-11-22T17:16:50,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742303_1479 (size=17181) 2024-11-22T17:16:50,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295810915, max timestamp=1732295810950Num uniques 859; 2024-11-22T17:16:50,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295810915, max timestamp=1732295810950Num uniques 859; 2024-11-22T17:16:50,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:16:50,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:16:50,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295810915, max timestamp=1732295810950Num uniques 859; 2024-11-22T17:16:50,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:16:51,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295871140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:51,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:51,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:51,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:51,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/92cff8478a4f4b00b3cb79dca278cbad 2024-11-22T17:16:51,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/bfd4c96bdc97418198a1640339df4595 is 50, key is test_row_0/B:col10/1732295810767/Put/seqid=0 2024-11-22T17:16:51,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742304_1480 (size=12301) 2024-11-22T17:16:51,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295871261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:51,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295871481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:51,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/bfd4c96bdc97418198a1640339df4595 2024-11-22T17:16:51,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/6daac954ba42438c8a9a4ffc10849963 is 50, key is test_row_0/C:col10/1732295810767/Put/seqid=0 2024-11-22T17:16:51,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742305_1481 (size=12301) 2024-11-22T17:16:51,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295871801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:52,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/6daac954ba42438c8a9a4ffc10849963 2024-11-22T17:16:52,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/92cff8478a4f4b00b3cb79dca278cbad as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/92cff8478a4f4b00b3cb79dca278cbad 2024-11-22T17:16:52,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/92cff8478a4f4b00b3cb79dca278cbad, entries=250, sequenceid=301, filesize=16.8 K 2024-11-22T17:16:52,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/bfd4c96bdc97418198a1640339df4595 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/bfd4c96bdc97418198a1640339df4595 2024-11-22T17:16:52,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/bfd4c96bdc97418198a1640339df4595, entries=150, sequenceid=301, filesize=12.0 K 2024-11-22T17:16:52,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/6daac954ba42438c8a9a4ffc10849963 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/6daac954ba42438c8a9a4ffc10849963 2024-11-22T17:16:52,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/6daac954ba42438c8a9a4ffc10849963, entries=150, sequenceid=301, filesize=12.0 K 2024-11-22T17:16:52,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5768c661d3c58ac27b37c9e997559f2f in 1278ms, sequenceid=301, compaction requested=true 2024-11-22T17:16:52,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:52,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:16:52,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:52,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:16:52,054 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:16:52,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:52,054 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:16:52,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5768c661d3c58ac27b37c9e997559f2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:16:52,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:52,055 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:16:52,055 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:16:52,055 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/B is initiating minor compaction (all files) 2024-11-22T17:16:52,055 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/A is initiating minor compaction (all files) 2024-11-22T17:16:52,055 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/B in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:52,055 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/A in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:52,055 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/f16a9560ac9e4de5a51a098b5021e03d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/2667ad9773c54976be7c45d968168c7c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/d4c979874923495f9d70e648165e2b0c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/bfd4c96bdc97418198a1640339df4595] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=48.5 K 2024-11-22T17:16:52,055 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a6628c0bb3194ae2a69b5703035261bb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/d24ed07446f04781b2ceb5dd04a0e130, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/d09d9ba111504b8396a084fbb4fe70d9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/92cff8478a4f4b00b3cb79dca278cbad] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=55.7 K 2024-11-22T17:16:52,055 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6628c0bb3194ae2a69b5703035261bb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732295806909 2024-11-22T17:16:52,055 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting f16a9560ac9e4de5a51a098b5021e03d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732295806909 2024-11-22T17:16:52,055 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting d24ed07446f04781b2ceb5dd04a0e130, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732295807279 2024-11-22T17:16:52,055 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 2667ad9773c54976be7c45d968168c7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732295807298 2024-11-22T17:16:52,055 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting d09d9ba111504b8396a084fbb4fe70d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732295808548 2024-11-22T17:16:52,055 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting d4c979874923495f9d70e648165e2b0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732295808548 2024-11-22T17:16:52,056 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92cff8478a4f4b00b3cb79dca278cbad, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732295810755 2024-11-22T17:16:52,056 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting bfd4c96bdc97418198a1640339df4595, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732295810755 2024-11-22T17:16:52,063 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#B#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:52,063 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#A#compaction#408 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:52,063 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/c64fda977feb4980bde0603b8b66a906 is 50, key is test_row_0/B:col10/1732295810767/Put/seqid=0 2024-11-22T17:16:52,063 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/1e561d47033f44ca9e202660c7a68463 is 50, key is test_row_0/A:col10/1732295810767/Put/seqid=0 2024-11-22T17:16:52,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742307_1483 (size=13085) 2024-11-22T17:16:52,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742306_1482 (size=13085) 2024-11-22T17:16:52,073 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/1e561d47033f44ca9e202660c7a68463 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/1e561d47033f44ca9e202660c7a68463 2024-11-22T17:16:52,077 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/A of 5768c661d3c58ac27b37c9e997559f2f into 1e561d47033f44ca9e202660c7a68463(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:52,077 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:52,077 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/A, priority=12, startTime=1732295812054; duration=0sec 2024-11-22T17:16:52,077 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:16:52,077 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:A 2024-11-22T17:16:52,077 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:16:52,078 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:16:52,078 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 5768c661d3c58ac27b37c9e997559f2f/C is initiating minor compaction (all files) 2024-11-22T17:16:52,078 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5768c661d3c58ac27b37c9e997559f2f/C in TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:52,078 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/61a3991f66a14540b04c7b9f99e07be2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/c80f2b1280c547779b6e5152fdcae7b1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/6207817291034fd0858963bf999f4125, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/6daac954ba42438c8a9a4ffc10849963] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp, totalSize=48.5 K 2024-11-22T17:16:52,078 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61a3991f66a14540b04c7b9f99e07be2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732295806909 2024-11-22T17:16:52,079 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting c80f2b1280c547779b6e5152fdcae7b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732295807298 2024-11-22T17:16:52,079 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6207817291034fd0858963bf999f4125, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732295808548 2024-11-22T17:16:52,079 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6daac954ba42438c8a9a4ffc10849963, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732295810755 2024-11-22T17:16:52,084 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5768c661d3c58ac27b37c9e997559f2f#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:16:52,084 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/48c6facdf1ef4cd2ae8b53b348f29758 is 50, key is test_row_0/C:col10/1732295810767/Put/seqid=0 2024-11-22T17:16:52,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742308_1484 (size=13085) 2024-11-22T17:16:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:52,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-22T17:16:52,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:52,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:16:52,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:52,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:16:52,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:52,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:16:52,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/a65cdfb301304092a033b8fa06924afd is 50, key is test_row_0/A:col10/1732295811138/Put/seqid=0 2024-11-22T17:16:52,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742309_1485 (size=14741) 2024-11-22T17:16:52,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295872378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:52,476 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/c64fda977feb4980bde0603b8b66a906 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/c64fda977feb4980bde0603b8b66a906 2024-11-22T17:16:52,480 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/B of 5768c661d3c58ac27b37c9e997559f2f into c64fda977feb4980bde0603b8b66a906(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:52,480 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:52,480 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/B, priority=12, startTime=1732295812054; duration=0sec 2024-11-22T17:16:52,480 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:52,480 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:B 2024-11-22T17:16:52,493 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/48c6facdf1ef4cd2ae8b53b348f29758 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/48c6facdf1ef4cd2ae8b53b348f29758 2024-11-22T17:16:52,497 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5768c661d3c58ac27b37c9e997559f2f/C of 5768c661d3c58ac27b37c9e997559f2f into 48c6facdf1ef4cd2ae8b53b348f29758(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:16:52,497 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:52,497 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., storeName=5768c661d3c58ac27b37c9e997559f2f/C, priority=12, startTime=1732295812054; duration=0sec 2024-11-22T17:16:52,497 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:16:52,497 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5768c661d3c58ac27b37c9e997559f2f:C 2024-11-22T17:16:52,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295872491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T17:16:52,562 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:52,562 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:16:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-22T17:16:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T17:16:52,564 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:16:52,564 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:16:52,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:16:52,601 INFO [Thread-1888 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1888. 2024-11-22T17:16:52,601 DEBUG [Thread-1888 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:52,601 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:52,601 INFO [Thread-1880 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1880. 2024-11-22T17:16:52,601 DEBUG [Thread-1880 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:52,601 DEBUG [Thread-1880 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:52,601 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:52,601 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:52,602 DEBUG [Thread-1880 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:52,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:52,606 INFO [Thread-1884 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1884. 2024-11-22T17:16:52,606 DEBUG [Thread-1884 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:52,606 DEBUG [Thread-1884 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:52,606 DEBUG [Thread-1884 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:52,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:52,613 INFO [Thread-1886 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1886. 2024-11-22T17:16:52,613 DEBUG [Thread-1886 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:52,613 DEBUG [Thread-1886 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:52,613 DEBUG [Thread-1886 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:52,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:52,614 INFO [Thread-1882 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1882. 2024-11-22T17:16:52,614 DEBUG [Thread-1882 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:52,614 DEBUG [Thread-1882 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:52,614 DEBUG [Thread-1882 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:52,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T17:16:52,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295872711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:52,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:52,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:52,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:52,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T17:16:52,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:52,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:52,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:52,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:52,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:52,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/a65cdfb301304092a033b8fa06924afd 2024-11-22T17:16:52,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/64860d8dacea4cf798d6f47e7af5a40d is 50, key is test_row_0/B:col10/1732295811138/Put/seqid=0 2024-11-22T17:16:52,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742310_1486 (size=12301) 2024-11-22T17:16:52,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T17:16:52,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:52,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:52,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:52,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:52,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T17:16:53,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:16:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48206 deadline: 1732295873021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 , the old value is region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:16:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., hostname=e61df7af61d2,42253,1732295655623, seqNum=2 because the exception is null or not the one we care about 2024-11-22T17:16:53,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T17:16:53,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:53,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:53,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:53,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:53,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:53,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/64860d8dacea4cf798d6f47e7af5a40d 2024-11-22T17:16:53,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/d18f75286bb943cab385ccf10f96e5b4 is 50, key is test_row_0/C:col10/1732295811138/Put/seqid=0 2024-11-22T17:16:53,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742311_1487 (size=12301) 2024-11-22T17:16:53,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/d18f75286bb943cab385ccf10f96e5b4 2024-11-22T17:16:53,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/a65cdfb301304092a033b8fa06924afd as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a65cdfb301304092a033b8fa06924afd 2024-11-22T17:16:53,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T17:16:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:53,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:16:53,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a65cdfb301304092a033b8fa06924afd, entries=200, sequenceid=339, filesize=14.4 K 2024-11-22T17:16:53,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/64860d8dacea4cf798d6f47e7af5a40d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/64860d8dacea4cf798d6f47e7af5a40d 2024-11-22T17:16:53,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/64860d8dacea4cf798d6f47e7af5a40d, entries=150, sequenceid=339, filesize=12.0 K 2024-11-22T17:16:53,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/d18f75286bb943cab385ccf10f96e5b4 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d18f75286bb943cab385ccf10f96e5b4 2024-11-22T17:16:53,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d18f75286bb943cab385ccf10f96e5b4, entries=150, sequenceid=339, filesize=12.0 K 2024-11-22T17:16:53,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 5768c661d3c58ac27b37c9e997559f2f in 855ms, sequenceid=339, compaction requested=false 2024-11-22T17:16:53,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:53,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T17:16:53,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T17:16:53,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:53,330 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-22T17:16:53,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:53,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:53,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:53,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:53,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:53,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:53,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/42b0803ffa0d45da9ce1254de9ece269 is 50, key is test_row_0/A:col10/1732295812356/Put/seqid=0 2024-11-22T17:16:53,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742312_1488 (size=12301) 2024-11-22T17:16:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:53,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. as already flushing 2024-11-22T17:16:53,532 INFO [Thread-1873 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1873. 2024-11-22T17:16:53,532 DEBUG [Thread-1873 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:53,532 DEBUG [Thread-1873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:53,532 DEBUG [Thread-1873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:53,532 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T17:16:53,743 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/42b0803ffa0d45da9ce1254de9ece269 2024-11-22T17:16:53,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/a25b548c70b14a389d7e27bb22f99d84 is 50, key is test_row_0/B:col10/1732295812356/Put/seqid=0 2024-11-22T17:16:53,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742313_1489 (size=12301) 2024-11-22T17:16:54,154 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/a25b548c70b14a389d7e27bb22f99d84 2024-11-22T17:16:54,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/94d98b318efc4b93b56e3abea157e431 is 50, key is test_row_0/C:col10/1732295812356/Put/seqid=0 2024-11-22T17:16:54,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742314_1490 (size=12301) 2024-11-22T17:16:54,569 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/94d98b318efc4b93b56e3abea157e431 2024-11-22T17:16:54,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/42b0803ffa0d45da9ce1254de9ece269 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/42b0803ffa0d45da9ce1254de9ece269 2024-11-22T17:16:54,578 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/42b0803ffa0d45da9ce1254de9ece269, entries=150, sequenceid=350, filesize=12.0 K 2024-11-22T17:16:54,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/a25b548c70b14a389d7e27bb22f99d84 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/a25b548c70b14a389d7e27bb22f99d84 2024-11-22T17:16:54,583 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/a25b548c70b14a389d7e27bb22f99d84, entries=150, sequenceid=350, filesize=12.0 K 2024-11-22T17:16:54,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/94d98b318efc4b93b56e3abea157e431 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/94d98b318efc4b93b56e3abea157e431 2024-11-22T17:16:54,588 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/94d98b318efc4b93b56e3abea157e431, entries=150, sequenceid=350, filesize=12.0 K 2024-11-22T17:16:54,589 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=6.71 KB/6870 for 5768c661d3c58ac27b37c9e997559f2f in 1259ms, sequenceid=350, compaction requested=true 2024-11-22T17:16:54,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 5768c661d3c58ac27b37c9e997559f2f: 2024-11-22T17:16:54,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:54,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-22T17:16:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-22T17:16:54,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-22T17:16:54,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0250 sec 2024-11-22T17:16:54,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.0310 sec 2024-11-22T17:16:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T17:16:54,712 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:56,205 INFO [Thread-1871 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1871. 2024-11-22T17:16:56,205 DEBUG [Thread-1871 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:56,205 DEBUG [Thread-1871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:56,206 DEBUG [Thread-1871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:56,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:56,225 INFO [Thread-1877 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1877. 2024-11-22T17:16:56,225 DEBUG [Thread-1877 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:56,225 DEBUG [Thread-1877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:56,225 DEBUG [Thread-1877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:56,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:56,243 INFO [Thread-1875 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1875. 2024-11-22T17:16:56,243 DEBUG [Thread-1875 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:56,243 DEBUG [Thread-1875 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:56,243 DEBUG [Thread-1875 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:56,243 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:56,284 INFO [Thread-1869 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1869. 2024-11-22T17:16:56,284 DEBUG [Thread-1869 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:16:56,284 DEBUG [Thread-1869 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:56,284 DEBUG [Thread-1869 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:56,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 195 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1133 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3397 rows 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1136 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3403 rows 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1139 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3412 rows 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1137 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3409 rows 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1133 2024-11-22T17:16:56,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3399 rows 2024-11-22T17:16:56,285 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T17:16:56,286 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T17:16:56,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:56,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:16:56,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:16:56,287 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T17:16:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T17:16:56,293 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295816292"}]},"ts":"1732295816292"} 2024-11-22T17:16:56,294 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T17:16:56,294 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T17:16:56,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T17:16:56,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5768c661d3c58ac27b37c9e997559f2f, UNASSIGN}] 2024-11-22T17:16:56,298 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5768c661d3c58ac27b37c9e997559f2f, UNASSIGN 2024-11-22T17:16:56,299 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=5768c661d3c58ac27b37c9e997559f2f, regionState=CLOSING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:56,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5768c661d3c58ac27b37c9e997559f2f, UNASSIGN because future has completed 2024-11-22T17:16:56,302 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T17:16:56,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:16:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T17:16:56,455 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:56,455 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T17:16:56,455 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing 5768c661d3c58ac27b37c9e997559f2f, disabling compactions & flushes 2024-11-22T17:16:56,455 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:56,455 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:56,455 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. after waiting 0 ms 2024-11-22T17:16:56,455 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:56,455 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing 5768c661d3c58ac27b37c9e997559f2f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-22T17:16:56,455 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=A 2024-11-22T17:16:56,455 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:56,455 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=B 2024-11-22T17:16:56,455 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:56,455 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5768c661d3c58ac27b37c9e997559f2f, store=C 2024-11-22T17:16:56,455 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:16:56,458 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/889616c63cfa4f39b63ea2548c7e3887 is 50, key is test_row_0/A:col10/1732295816202/Put/seqid=0 2024-11-22T17:16:56,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742315_1491 (size=12301) 2024-11-22T17:16:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T17:16:56,862 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/889616c63cfa4f39b63ea2548c7e3887 2024-11-22T17:16:56,869 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/6bb7d3c07f4c4490b42c34d7b9590f0e is 50, key is test_row_0/B:col10/1732295816202/Put/seqid=0 2024-11-22T17:16:56,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742316_1492 (size=12301) 2024-11-22T17:16:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T17:16:57,273 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/6bb7d3c07f4c4490b42c34d7b9590f0e 2024-11-22T17:16:57,280 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/e009bf45a28644f6bb4683f50e99beeb is 50, key is test_row_0/C:col10/1732295816202/Put/seqid=0 2024-11-22T17:16:57,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742317_1493 (size=12301) 2024-11-22T17:16:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T17:16:57,685 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/e009bf45a28644f6bb4683f50e99beeb 2024-11-22T17:16:57,692 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/A/889616c63cfa4f39b63ea2548c7e3887 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/889616c63cfa4f39b63ea2548c7e3887 2024-11-22T17:16:57,698 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/889616c63cfa4f39b63ea2548c7e3887, entries=150, sequenceid=358, filesize=12.0 K 2024-11-22T17:16:57,699 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/B/6bb7d3c07f4c4490b42c34d7b9590f0e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/6bb7d3c07f4c4490b42c34d7b9590f0e 2024-11-22T17:16:57,703 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/6bb7d3c07f4c4490b42c34d7b9590f0e, entries=150, sequenceid=358, filesize=12.0 K 2024-11-22T17:16:57,704 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/.tmp/C/e009bf45a28644f6bb4683f50e99beeb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/e009bf45a28644f6bb4683f50e99beeb 2024-11-22T17:16:57,707 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/e009bf45a28644f6bb4683f50e99beeb, entries=150, sequenceid=358, filesize=12.0 K 2024-11-22T17:16:57,708 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5768c661d3c58ac27b37c9e997559f2f in 1253ms, sequenceid=358, compaction requested=true 2024-11-22T17:16:57,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/7537331f0b8548c5af08cb2683d82b50, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/027ee03fd5164f80a71ed54d2b6a178d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a9f13562b32d4e5cbc6d145b2f6ff507, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/10e75d6ca07b4caaadce20a5bd2503ba, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/31c9b2df806d4c0baca76f2ca487ac30, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/ec0e663dd9ae4dc3a5fd0b46f0cb3cb5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/fd250b2891c543c79d588f97d72d16ba, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/f6d45c6618ea4ee3b57797fafa4116c6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/0fd175a9db9b4a2c8aead5a17ec132f2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bfe501fa864047c884b26bbca2d32baf, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a656f1889b8a4dd5b833dab4610e90aa, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/aaf1b2f9473a4a6db95c52162aa83349, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/db23c172315148efaadc14e48fdd9fca, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bcf05b06b96341a59454376bdbfc98bb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/af585ddf45614a169eab215fe6774638, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/b0ab7b96c2f540048b2fd4e7068d0b4b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/f1276ac9b75a4e26a87f437c457cab31, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/91bc18f880e44281aabb49536f6e85e5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bc6c27c138ff41bd822b963b8c700a8b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a6628c0bb3194ae2a69b5703035261bb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/d24ed07446f04781b2ceb5dd04a0e130, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/d09d9ba111504b8396a084fbb4fe70d9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/92cff8478a4f4b00b3cb79dca278cbad] to archive 2024-11-22T17:16:57,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:16:57,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/7537331f0b8548c5af08cb2683d82b50 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/7537331f0b8548c5af08cb2683d82b50 2024-11-22T17:16:57,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/027ee03fd5164f80a71ed54d2b6a178d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/027ee03fd5164f80a71ed54d2b6a178d 2024-11-22T17:16:57,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a9f13562b32d4e5cbc6d145b2f6ff507 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a9f13562b32d4e5cbc6d145b2f6ff507 2024-11-22T17:16:57,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/10e75d6ca07b4caaadce20a5bd2503ba to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/10e75d6ca07b4caaadce20a5bd2503ba 2024-11-22T17:16:57,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/31c9b2df806d4c0baca76f2ca487ac30 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/31c9b2df806d4c0baca76f2ca487ac30 2024-11-22T17:16:57,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/ec0e663dd9ae4dc3a5fd0b46f0cb3cb5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/ec0e663dd9ae4dc3a5fd0b46f0cb3cb5 2024-11-22T17:16:57,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/fd250b2891c543c79d588f97d72d16ba to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/fd250b2891c543c79d588f97d72d16ba 2024-11-22T17:16:57,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/f6d45c6618ea4ee3b57797fafa4116c6 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/f6d45c6618ea4ee3b57797fafa4116c6 2024-11-22T17:16:57,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/0fd175a9db9b4a2c8aead5a17ec132f2 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/0fd175a9db9b4a2c8aead5a17ec132f2 2024-11-22T17:16:57,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bfe501fa864047c884b26bbca2d32baf to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bfe501fa864047c884b26bbca2d32baf 2024-11-22T17:16:57,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a656f1889b8a4dd5b833dab4610e90aa to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a656f1889b8a4dd5b833dab4610e90aa 2024-11-22T17:16:57,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/aaf1b2f9473a4a6db95c52162aa83349 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/aaf1b2f9473a4a6db95c52162aa83349 2024-11-22T17:16:57,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/db23c172315148efaadc14e48fdd9fca to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/db23c172315148efaadc14e48fdd9fca 2024-11-22T17:16:57,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bcf05b06b96341a59454376bdbfc98bb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bcf05b06b96341a59454376bdbfc98bb 2024-11-22T17:16:57,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/af585ddf45614a169eab215fe6774638 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/af585ddf45614a169eab215fe6774638 2024-11-22T17:16:57,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/b0ab7b96c2f540048b2fd4e7068d0b4b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/b0ab7b96c2f540048b2fd4e7068d0b4b 2024-11-22T17:16:57,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/f1276ac9b75a4e26a87f437c457cab31 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/f1276ac9b75a4e26a87f437c457cab31 2024-11-22T17:16:57,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/91bc18f880e44281aabb49536f6e85e5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/91bc18f880e44281aabb49536f6e85e5 2024-11-22T17:16:57,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bc6c27c138ff41bd822b963b8c700a8b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/bc6c27c138ff41bd822b963b8c700a8b 2024-11-22T17:16:57,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a6628c0bb3194ae2a69b5703035261bb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a6628c0bb3194ae2a69b5703035261bb 2024-11-22T17:16:57,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/d24ed07446f04781b2ceb5dd04a0e130 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/d24ed07446f04781b2ceb5dd04a0e130 2024-11-22T17:16:57,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/d09d9ba111504b8396a084fbb4fe70d9 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/d09d9ba111504b8396a084fbb4fe70d9 2024-11-22T17:16:57,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/92cff8478a4f4b00b3cb79dca278cbad to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/92cff8478a4f4b00b3cb79dca278cbad 2024-11-22T17:16:57,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/5b162f4646134d1ba3bca2cbc7382c6e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/1d897da2ddf244fcad90d1622329a1d2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/61e3adeb1e72409296a79bb43904d94d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/fcf5b1e9b67748e185231197c5879cf6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/ae0d78d4b4274cdab33203824826e7ab, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/523a7cb4490b453b9a29d12e50ba3c79, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/8a00b47f51a94ba59f1bff5b17a42e33, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/928150a051984a63bee80d755aa84d29, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e2df5b8b9916401e99ed6cdee97950f2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/cbb0e03fa3724f61ab4dc189a3947c99, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/d49f92121d5a4b0bada8c6d635fdf7f4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/c711ab2f050740c292d92900667a4a39, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/1e47925ad4594e519e2455d4e720ed9e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/0c9189a2576647d7bdd55e55f9a868d5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e768c1271ae645458e599089a562c562, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/9113b2f8bf284478b44313a55fe907ae, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e53eb24472f34d99a89e99e937583d14, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/11709cf1b1844f3e852bcc30467afca6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/f16a9560ac9e4de5a51a098b5021e03d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/b720947529814553a1782d5e5b64403c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/2667ad9773c54976be7c45d968168c7c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/d4c979874923495f9d70e648165e2b0c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/bfd4c96bdc97418198a1640339df4595] to archive 2024-11-22T17:16:57,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:16:57,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/5b162f4646134d1ba3bca2cbc7382c6e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/5b162f4646134d1ba3bca2cbc7382c6e 2024-11-22T17:16:57,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/1d897da2ddf244fcad90d1622329a1d2 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/1d897da2ddf244fcad90d1622329a1d2 2024-11-22T17:16:57,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/61e3adeb1e72409296a79bb43904d94d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/61e3adeb1e72409296a79bb43904d94d 2024-11-22T17:16:57,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/fcf5b1e9b67748e185231197c5879cf6 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/fcf5b1e9b67748e185231197c5879cf6 2024-11-22T17:16:57,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/ae0d78d4b4274cdab33203824826e7ab to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/ae0d78d4b4274cdab33203824826e7ab 2024-11-22T17:16:57,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/523a7cb4490b453b9a29d12e50ba3c79 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/523a7cb4490b453b9a29d12e50ba3c79 2024-11-22T17:16:57,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/8a00b47f51a94ba59f1bff5b17a42e33 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/8a00b47f51a94ba59f1bff5b17a42e33 2024-11-22T17:16:57,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/928150a051984a63bee80d755aa84d29 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/928150a051984a63bee80d755aa84d29 2024-11-22T17:16:57,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e2df5b8b9916401e99ed6cdee97950f2 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e2df5b8b9916401e99ed6cdee97950f2 2024-11-22T17:16:57,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/cbb0e03fa3724f61ab4dc189a3947c99 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/cbb0e03fa3724f61ab4dc189a3947c99 2024-11-22T17:16:57,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/d49f92121d5a4b0bada8c6d635fdf7f4 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/d49f92121d5a4b0bada8c6d635fdf7f4 2024-11-22T17:16:57,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/c711ab2f050740c292d92900667a4a39 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/c711ab2f050740c292d92900667a4a39 2024-11-22T17:16:57,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/1e47925ad4594e519e2455d4e720ed9e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/1e47925ad4594e519e2455d4e720ed9e 2024-11-22T17:16:57,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/0c9189a2576647d7bdd55e55f9a868d5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/0c9189a2576647d7bdd55e55f9a868d5 2024-11-22T17:16:57,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e768c1271ae645458e599089a562c562 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e768c1271ae645458e599089a562c562 2024-11-22T17:16:57,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/9113b2f8bf284478b44313a55fe907ae to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/9113b2f8bf284478b44313a55fe907ae 2024-11-22T17:16:57,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e53eb24472f34d99a89e99e937583d14 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/e53eb24472f34d99a89e99e937583d14 2024-11-22T17:16:57,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/11709cf1b1844f3e852bcc30467afca6 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/11709cf1b1844f3e852bcc30467afca6 2024-11-22T17:16:57,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/f16a9560ac9e4de5a51a098b5021e03d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/f16a9560ac9e4de5a51a098b5021e03d 2024-11-22T17:16:57,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/b720947529814553a1782d5e5b64403c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/b720947529814553a1782d5e5b64403c 2024-11-22T17:16:57,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/2667ad9773c54976be7c45d968168c7c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/2667ad9773c54976be7c45d968168c7c 2024-11-22T17:16:57,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/d4c979874923495f9d70e648165e2b0c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/d4c979874923495f9d70e648165e2b0c 2024-11-22T17:16:57,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/bfd4c96bdc97418198a1640339df4595 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/bfd4c96bdc97418198a1640339df4595 2024-11-22T17:16:57,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/2a905d4fb8b747a0935829471ae15e04, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/fd0eb548e9bc485fb0256a0b4b545f49, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/e453df60edac464c9c9c38673e521292, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/7f50e66e1fb6499f822285163aa3519b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/96b440349d5f404d957781cb6f9d7ae6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/3c0418453fa1439ca0bbe781111be85b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/b69a4c1c260c4800bf6c47cc3b388a2a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d307f3cdcc16447881c700443583d19c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d1182b92cccb449db84bb7558d9fd662, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/cc1cecb76c634a9bbfef2e08ef61a653, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/027cef4a6bf34effbb146f410bf36d03, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/a65745f04854406a972d5df32619881e, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/0cce11c53d5a44cfb6a3be4f1dfdb673, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/488961a06acd41839bc71002b8a3a3e5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/a6cc1bad0478467abcdd11ddc3cfece1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/9fa8585bc60c4606855782a556ba8b4d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/da3fbaa786ec4692aa579ebeb0c173a4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/576c4c46d44940059ac50728bb98e836, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/61a3991f66a14540b04c7b9f99e07be2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/1f8fe8f4ee884c8d99da3f653672475b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/c80f2b1280c547779b6e5152fdcae7b1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/6207817291034fd0858963bf999f4125, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/6daac954ba42438c8a9a4ffc10849963] to archive 2024-11-22T17:16:57,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:16:57,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/2a905d4fb8b747a0935829471ae15e04 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/2a905d4fb8b747a0935829471ae15e04 2024-11-22T17:16:57,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/fd0eb548e9bc485fb0256a0b4b545f49 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/fd0eb548e9bc485fb0256a0b4b545f49 2024-11-22T17:16:57,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/e453df60edac464c9c9c38673e521292 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/e453df60edac464c9c9c38673e521292 2024-11-22T17:16:57,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/7f50e66e1fb6499f822285163aa3519b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/7f50e66e1fb6499f822285163aa3519b 2024-11-22T17:16:57,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/96b440349d5f404d957781cb6f9d7ae6 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/96b440349d5f404d957781cb6f9d7ae6 2024-11-22T17:16:57,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/3c0418453fa1439ca0bbe781111be85b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/3c0418453fa1439ca0bbe781111be85b 2024-11-22T17:16:57,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/b69a4c1c260c4800bf6c47cc3b388a2a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/b69a4c1c260c4800bf6c47cc3b388a2a 2024-11-22T17:16:57,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d307f3cdcc16447881c700443583d19c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d307f3cdcc16447881c700443583d19c 2024-11-22T17:16:57,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d1182b92cccb449db84bb7558d9fd662 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d1182b92cccb449db84bb7558d9fd662 2024-11-22T17:16:57,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/cc1cecb76c634a9bbfef2e08ef61a653 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/cc1cecb76c634a9bbfef2e08ef61a653 2024-11-22T17:16:57,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/027cef4a6bf34effbb146f410bf36d03 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/027cef4a6bf34effbb146f410bf36d03 2024-11-22T17:16:57,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/a65745f04854406a972d5df32619881e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/a65745f04854406a972d5df32619881e 2024-11-22T17:16:57,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/0cce11c53d5a44cfb6a3be4f1dfdb673 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/0cce11c53d5a44cfb6a3be4f1dfdb673 2024-11-22T17:16:57,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/488961a06acd41839bc71002b8a3a3e5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/488961a06acd41839bc71002b8a3a3e5 2024-11-22T17:16:57,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/a6cc1bad0478467abcdd11ddc3cfece1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/a6cc1bad0478467abcdd11ddc3cfece1 2024-11-22T17:16:57,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/9fa8585bc60c4606855782a556ba8b4d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/9fa8585bc60c4606855782a556ba8b4d 2024-11-22T17:16:57,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/da3fbaa786ec4692aa579ebeb0c173a4 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/da3fbaa786ec4692aa579ebeb0c173a4 2024-11-22T17:16:57,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/576c4c46d44940059ac50728bb98e836 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/576c4c46d44940059ac50728bb98e836 2024-11-22T17:16:57,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/61a3991f66a14540b04c7b9f99e07be2 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/61a3991f66a14540b04c7b9f99e07be2 2024-11-22T17:16:57,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/1f8fe8f4ee884c8d99da3f653672475b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/1f8fe8f4ee884c8d99da3f653672475b 2024-11-22T17:16:57,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/c80f2b1280c547779b6e5152fdcae7b1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/c80f2b1280c547779b6e5152fdcae7b1 2024-11-22T17:16:57,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/6207817291034fd0858963bf999f4125 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/6207817291034fd0858963bf999f4125 2024-11-22T17:16:57,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/6daac954ba42438c8a9a4ffc10849963 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/6daac954ba42438c8a9a4ffc10849963 2024-11-22T17:16:57,798 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/recovered.edits/361.seqid, newMaxSeqId=361, maxSeqId=1 2024-11-22T17:16:57,798 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. 2024-11-22T17:16:57,799 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for 5768c661d3c58ac27b37c9e997559f2f: Waiting for close lock at 1732295816455Running coprocessor pre-close hooks at 1732295816455Disabling compacts and flushes for region at 1732295816455Disabling writes for close at 1732295816455Obtaining lock to block concurrent updates at 1732295816455Preparing flush snapshotting stores in 5768c661d3c58ac27b37c9e997559f2f at 1732295816455Finished memstore snapshotting TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732295816455Flushing stores of TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f. at 1732295816456 (+1 ms)Flushing 5768c661d3c58ac27b37c9e997559f2f/A: creating writer at 1732295816456Flushing 5768c661d3c58ac27b37c9e997559f2f/A: appending metadata at 1732295816458 (+2 ms)Flushing 5768c661d3c58ac27b37c9e997559f2f/A: closing flushed file at 1732295816458Flushing 5768c661d3c58ac27b37c9e997559f2f/B: creating writer at 1732295816866 (+408 ms)Flushing 5768c661d3c58ac27b37c9e997559f2f/B: appending metadata at 1732295816868 (+2 ms)Flushing 5768c661d3c58ac27b37c9e997559f2f/B: closing flushed file at 1732295816868Flushing 5768c661d3c58ac27b37c9e997559f2f/C: creating writer at 1732295817277 (+409 ms)Flushing 5768c661d3c58ac27b37c9e997559f2f/C: appending metadata at 1732295817280 (+3 ms)Flushing 5768c661d3c58ac27b37c9e997559f2f/C: closing flushed file at 1732295817280Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bb42727: reopening flushed file at 1732295817691 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60e3c3f9: reopening flushed file at 1732295817698 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fb73874: reopening flushed file at 1732295817703 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5768c661d3c58ac27b37c9e997559f2f in 1253ms, sequenceid=358, compaction requested=true at 1732295817708 (+5 ms)Writing region close event to WAL at 1732295817794 (+86 ms)Running coprocessor post-close hooks at 1732295817798 (+4 ms)Closed at 1732295817798 2024-11-22T17:16:57,801 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed 5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:57,801 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=5768c661d3c58ac27b37c9e997559f2f, regionState=CLOSED 2024-11-22T17:16:57,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:16:57,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-22T17:16:57,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure 5768c661d3c58ac27b37c9e997559f2f, server=e61df7af61d2,42253,1732295655623 in 1.5020 sec 2024-11-22T17:16:57,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-22T17:16:57,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5768c661d3c58ac27b37c9e997559f2f, UNASSIGN in 1.5090 sec 2024-11-22T17:16:57,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-22T17:16:57,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5130 sec 2024-11-22T17:16:57,810 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295817810"}]},"ts":"1732295817810"} 2024-11-22T17:16:57,812 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T17:16:57,812 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T17:16:57,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5250 sec 2024-11-22T17:16:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T17:16:58,442 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:58,443 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T17:16:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:58,447 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T17:16:58,449 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:58,453 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:58,455 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/recovered.edits] 2024-11-22T17:16:58,458 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/1e561d47033f44ca9e202660c7a68463 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/1e561d47033f44ca9e202660c7a68463 2024-11-22T17:16:58,459 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/42b0803ffa0d45da9ce1254de9ece269 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/42b0803ffa0d45da9ce1254de9ece269 2024-11-22T17:16:58,460 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/889616c63cfa4f39b63ea2548c7e3887 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/889616c63cfa4f39b63ea2548c7e3887 2024-11-22T17:16:58,461 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a65cdfb301304092a033b8fa06924afd to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/A/a65cdfb301304092a033b8fa06924afd 2024-11-22T17:16:58,462 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/64860d8dacea4cf798d6f47e7af5a40d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/64860d8dacea4cf798d6f47e7af5a40d 2024-11-22T17:16:58,463 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/6bb7d3c07f4c4490b42c34d7b9590f0e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/6bb7d3c07f4c4490b42c34d7b9590f0e 2024-11-22T17:16:58,463 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/a25b548c70b14a389d7e27bb22f99d84 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/a25b548c70b14a389d7e27bb22f99d84 2024-11-22T17:16:58,464 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/c64fda977feb4980bde0603b8b66a906 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/B/c64fda977feb4980bde0603b8b66a906 2024-11-22T17:16:58,465 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/48c6facdf1ef4cd2ae8b53b348f29758 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/48c6facdf1ef4cd2ae8b53b348f29758 2024-11-22T17:16:58,466 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/94d98b318efc4b93b56e3abea157e431 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/94d98b318efc4b93b56e3abea157e431 2024-11-22T17:16:58,467 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d18f75286bb943cab385ccf10f96e5b4 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/d18f75286bb943cab385ccf10f96e5b4 2024-11-22T17:16:58,467 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/e009bf45a28644f6bb4683f50e99beeb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/C/e009bf45a28644f6bb4683f50e99beeb 2024-11-22T17:16:58,469 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/recovered.edits/361.seqid to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f/recovered.edits/361.seqid 2024-11-22T17:16:58,469 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/5768c661d3c58ac27b37c9e997559f2f 2024-11-22T17:16:58,469 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T17:16:58,471 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:58,473 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T17:16:58,474 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T17:16:58,475 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:58,475 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T17:16:58,476 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732295818475"}]},"ts":"9223372036854775807"} 2024-11-22T17:16:58,478 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T17:16:58,478 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5768c661d3c58ac27b37c9e997559f2f, NAME => 'TestAcidGuarantees,,1732295790385.5768c661d3c58ac27b37c9e997559f2f.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T17:16:58,478 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T17:16:58,478 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732295818478"}]},"ts":"9223372036854775807"} 2024-11-22T17:16:58,479 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T17:16:58,480 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:58,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-11-22T17:16:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T17:16:58,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T17:16:58,554 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:16:58,560 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 249), OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=429 (was 320) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2607 (was 2007) - AvailableMemoryMB LEAK? - 2024-11-22T17:16:58,569 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=429, ProcessCount=11, AvailableMemoryMB=2606 2024-11-22T17:16:58,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T17:16:58,570 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T17:16:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T17:16:58,572 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T17:16:58,573 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:58,573 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-22T17:16:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-22T17:16:58,574 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T17:16:58,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742318_1494 (size=1061) 2024-11-22T17:16:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-22T17:16:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-22T17:16:58,980 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8 2024-11-22T17:16:58,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742319_1495 (size=53) 2024-11-22T17:16:59,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-22T17:16:59,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:16:59,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 83841b437d25c9378a29982f12a6cd16, disabling compactions & flushes 2024-11-22T17:16:59,387 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:16:59,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:16:59,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. after waiting 0 ms 2024-11-22T17:16:59,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:16:59,387 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:16:59,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 83841b437d25c9378a29982f12a6cd16: Waiting for close lock at 1732295819387Disabling compacts and flushes for region at 1732295819387Disabling writes for close at 1732295819387Writing region close event to WAL at 1732295819387Closed at 1732295819387 2024-11-22T17:16:59,388 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T17:16:59,388 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732295819388"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732295819388"}]},"ts":"1732295819388"} 2024-11-22T17:16:59,389 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T17:16:59,390 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T17:16:59,390 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295819390"}]},"ts":"1732295819390"} 2024-11-22T17:16:59,391 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T17:16:59,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, ASSIGN}] 2024-11-22T17:16:59,392 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, ASSIGN 2024-11-22T17:16:59,393 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, ASSIGN; state=OFFLINE, location=e61df7af61d2,42253,1732295655623; forceNewPlan=false, retain=false 2024-11-22T17:16:59,543 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=83841b437d25c9378a29982f12a6cd16, regionState=OPENING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:59,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, ASSIGN because future has completed 2024-11-22T17:16:59,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:16:59,700 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:16:59,700 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} 2024-11-22T17:16:59,700 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:16:59,701 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:16:59,701 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:16:59,701 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:16:59,702 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:16:59,703 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:16:59,703 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83841b437d25c9378a29982f12a6cd16 columnFamilyName A 2024-11-22T17:16:59,703 DEBUG [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:59,704 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.HStore(327): Store=83841b437d25c9378a29982f12a6cd16/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:16:59,704 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:16:59,705 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:16:59,705 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83841b437d25c9378a29982f12a6cd16 columnFamilyName B 2024-11-22T17:16:59,705 DEBUG [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:59,706 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.HStore(327): Store=83841b437d25c9378a29982f12a6cd16/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:16:59,706 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:16:59,707 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:16:59,707 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83841b437d25c9378a29982f12a6cd16 columnFamilyName C 2024-11-22T17:16:59,707 DEBUG [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:16:59,707 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.HStore(327): Store=83841b437d25c9378a29982f12a6cd16/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:16:59,707 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:16:59,707 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:16:59,708 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16 2024-11-22T17:16:59,708 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16 2024-11-22T17:16:59,709 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:16:59,709 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:16:59,710 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T17:16:59,711 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:16:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-22T17:16:59,712 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T17:16:59,713 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened 83841b437d25c9378a29982f12a6cd16; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64973442, jitterRate=-0.03182026743888855}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T17:16:59,713 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:16:59,713 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for 83841b437d25c9378a29982f12a6cd16: Running coprocessor pre-open hook at 1732295819701Writing region info on filesystem at 1732295819701Initializing all the Stores at 1732295819702 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295819702Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295819702Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295819702Cleaning up temporary data from old regions at 1732295819709 (+7 ms)Running coprocessor post-open hooks at 1732295819713 (+4 ms)Region opened successfully at 1732295819713 2024-11-22T17:16:59,714 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., pid=138, masterSystemTime=1732295819697 2024-11-22T17:16:59,715 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:16:59,715 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:16:59,716 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=83841b437d25c9378a29982f12a6cd16, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:16:59,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:16:59,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-22T17:16:59,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 in 174 msec 2024-11-22T17:16:59,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-22T17:16:59,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, ASSIGN in 329 msec 2024-11-22T17:16:59,723 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T17:16:59,723 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295819723"}]},"ts":"1732295819723"} 2024-11-22T17:16:59,724 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T17:16:59,725 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T17:16:59,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-22T17:17:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-22T17:17:00,722 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:17:00,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e17bf09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:00,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:17:00,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:17:00,723 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:17:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:17:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:17:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33e43321, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:17:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:17:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:00,725 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:17:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed58018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:00,726 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:17:00,727 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:17:00,728 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T17:17:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is e61df7af61d2,45693,1732295654407 2024-11-22T17:17:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23471467 2024-11-22T17:17:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T17:17:00,729 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T17:17:00,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T17:17:00,730 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T17:17:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T17:17:00,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742320_1496 (size=1097) 2024-11-22T17:17:00,741 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-22T17:17:00,741 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-22T17:17:00,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T17:17:00,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, REOPEN/MOVE}] 2024-11-22T17:17:00,747 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, REOPEN/MOVE 2024-11-22T17:17:00,748 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=83841b437d25c9378a29982f12a6cd16, regionState=CLOSING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:00,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, REOPEN/MOVE because future has completed 2024-11-22T17:17:00,749 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T17:17:00,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:17:00,902 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:00,902 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T17:17:00,902 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 83841b437d25c9378a29982f12a6cd16, disabling compactions & flushes 2024-11-22T17:17:00,902 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:00,902 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:00,902 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. after waiting 0 ms 2024-11-22T17:17:00,902 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:00,909 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T17:17:00,910 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:00,910 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 83841b437d25c9378a29982f12a6cd16: Waiting for close lock at 1732295820902Running coprocessor pre-close hooks at 1732295820902Disabling compacts and flushes for region at 1732295820902Disabling writes for close at 1732295820902Writing region close event to WAL at 1732295820903 (+1 ms)Running coprocessor post-close hooks at 1732295820910 (+7 ms)Closed at 1732295820910 2024-11-22T17:17:00,910 WARN [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: 83841b437d25c9378a29982f12a6cd16 to self. 2024-11-22T17:17:00,912 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:00,913 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=83841b437d25c9378a29982f12a6cd16, regionState=CLOSED 2024-11-22T17:17:00,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:17:00,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-22T17:17:00,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 in 165 msec 2024-11-22T17:17:00,916 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, REOPEN/MOVE; state=CLOSED, location=e61df7af61d2,42253,1732295655623; forceNewPlan=false, retain=true 2024-11-22T17:17:01,067 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=83841b437d25c9378a29982f12a6cd16, regionState=OPENING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, REOPEN/MOVE because future has completed 2024-11-22T17:17:01,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:17:01,224 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:01,224 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} 2024-11-22T17:17:01,225 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,225 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T17:17:01,225 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,225 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,226 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,227 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:17:01,227 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83841b437d25c9378a29982f12a6cd16 columnFamilyName A 2024-11-22T17:17:01,228 DEBUG [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:01,229 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.HStore(327): Store=83841b437d25c9378a29982f12a6cd16/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:17:01,229 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,230 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:17:01,230 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83841b437d25c9378a29982f12a6cd16 columnFamilyName B 2024-11-22T17:17:01,230 DEBUG [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:01,230 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.HStore(327): Store=83841b437d25c9378a29982f12a6cd16/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:17:01,230 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,231 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T17:17:01,231 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83841b437d25c9378a29982f12a6cd16 columnFamilyName C 2024-11-22T17:17:01,231 DEBUG [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:01,232 INFO [StoreOpener-83841b437d25c9378a29982f12a6cd16-1 {}] regionserver.HStore(327): Store=83841b437d25c9378a29982f12a6cd16/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T17:17:01,232 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:01,232 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,233 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,234 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,235 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,235 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,236 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T17:17:01,238 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,239 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened 83841b437d25c9378a29982f12a6cd16; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67041245, jitterRate=-0.0010076016187667847}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T17:17:01,239 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,240 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for 83841b437d25c9378a29982f12a6cd16: Running coprocessor pre-open hook at 1732295821225Writing region info on filesystem at 1732295821225Initializing all the Stores at 1732295821226 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295821226Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295821226Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732295821226Cleaning up temporary data from old regions at 1732295821235 (+9 ms)Running coprocessor post-open hooks at 1732295821239 (+4 ms)Region opened successfully at 1732295821240 (+1 ms) 2024-11-22T17:17:01,241 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., pid=143, masterSystemTime=1732295821221 2024-11-22T17:17:01,243 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:01,243 INFO [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:01,243 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=83841b437d25c9378a29982f12a6cd16, regionState=OPEN, openSeqNum=5, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:17:01,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-22T17:17:01,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 in 176 msec 2024-11-22T17:17:01,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-22T17:17:01,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, REOPEN/MOVE in 501 msec 2024-11-22T17:17:01,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-22T17:17:01,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 505 msec 2024-11-22T17:17:01,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 521 msec 2024-11-22T17:17:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T17:17:01,255 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:17:01,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac41996, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:17:01,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:17:01,257 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:17:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:17:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:17:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f7e587c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:17:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:17:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:01,259 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:17:01,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@482d693c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22aab9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:17:01,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:17:01,261 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:17:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:17:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:17:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a725b37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:17:01,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:17:01,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:01,263 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:17:01,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7b0922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a34ec22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:17:01,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:17:01,265 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:17:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:17:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:17:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ebe7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:17:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:17:01,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:01,266 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:17:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@632afd9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@380ce272, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:17:01,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:17:01,269 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:17:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:17:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:17:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e4f376c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:17:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:17:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:01,270 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:17:01,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3745f68c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c11d8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:17:01,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:17:01,273 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:17:01,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:17:01,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:17:01,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f59427, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:17:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:17:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:01,275 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:17:01,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ca5cafc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc99027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:17:01,276 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:17:01,277 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:17:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:17:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f91519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:01,279 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:17:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d82695f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24edf963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:17:01,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:17:01,281 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:17:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:17:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:17:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fbd236a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:17:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:17:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:01,283 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:17:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cc321e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2966c360, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:17:01,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:17:01,285 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5beb6022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:17:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:01,286 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:17:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2070e522, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9b51f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,287 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:17:01,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:17:01,288 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:17:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:17:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:17:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d588b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:17:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:17:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:01,289 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:17:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74e9a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac3ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45693,-1 for getting cluster id 2024-11-22T17:17:01,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T17:17:01,290 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10112fb3-27bd-4534-a278-6e2e2aff8d5f' 2024-11-22T17:17:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T17:17:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10112fb3-27bd-4534-a278-6e2e2aff8d5f" 2024-11-22T17:17:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d89b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45693,-1] 2024-11-22T17:17:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T17:17:01,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:01,291 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T17:17:01,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16e5eb69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T17:17:01,292 DEBUG [Thread-2238 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:17:01,292 DEBUG [Thread-2249 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:17:01,292 DEBUG [Thread-2253 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:17:01,292 DEBUG [Thread-2244 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:17:01,292 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:17:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:17:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:17:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:17:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:17:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:17:01,293 DEBUG [Thread-2257 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:17:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:17:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:17:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:17:01,293 DEBUG [Thread-2255 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:17:01,293 DEBUG [Thread-2242 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:17:01,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:17:01,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:17:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:17:01,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:17:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-22T17:17:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:17:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:17:01,294 DEBUG [Thread-2240 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:17:01,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:17:01,295 DEBUG [Thread-2246 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:17:01,295 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:17:01,295 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:17:01,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:17:01,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:17:01,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:17:01,296 DEBUG [Thread-2251 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T17:17:01,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:17:01,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:17:01,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:17:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:17:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,42253,1732295655623, seqNum=-1] 2024-11-22T17:17:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:17:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-22T17:17:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T17:17:01,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:17:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:17:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:17:01,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:17:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:17:01,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:17:01,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:17:01,300 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:17:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:17:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:17:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:17:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:17:01,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T17:17:01,302 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:17:01,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:17:01,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5] 2024-11-22T17:17:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:17:01,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:01,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:01,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:01,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:01,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:01,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:01,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295881333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295881335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295881337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a2e7cfd75f9349f8aa5b8a6b983fcfb2_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295821308/Put/seqid=0 2024-11-22T17:17:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295881338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295881339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742321_1497 (size=12154) 2024-11-22T17:17:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-22T17:17:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295881441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295881452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295881452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-22T17:17:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295881452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:01,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:01,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295881452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:01,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:01,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-22T17:17:01,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:01,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:01,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:01,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:01,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-22T17:17:01,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295881652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295881661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295881661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295881661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295881662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-22T17:17:01,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:01,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:01,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:01,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:01,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:01,769 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:01,772 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a2e7cfd75f9349f8aa5b8a6b983fcfb2_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a2e7cfd75f9349f8aa5b8a6b983fcfb2_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:01,772 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/750954e81ac34bd195aeeafaa32cc160, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:01,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/750954e81ac34bd195aeeafaa32cc160 is 175, key is test_row_0/A:col10/1732295821308/Put/seqid=0 2024-11-22T17:17:01,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742322_1498 (size=30955) 2024-11-22T17:17:01,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-22T17:17:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:01,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-22T17:17:01,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295881961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295881971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295881971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295881971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295881971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:02,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-22T17:17:02,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:02,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,177 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/750954e81ac34bd195aeeafaa32cc160 2024-11-22T17:17:02,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/da9ef9fba2ba4042bc44b925897744ce is 50, key is test_row_0/B:col10/1732295821308/Put/seqid=0 2024-11-22T17:17:02,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742323_1499 (size=12001) 2024-11-22T17:17:02,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-22T17:17:02,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:02,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-22T17:17:02,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:02,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-22T17:17:02,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295882471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:02,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295882481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:02,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295882481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:02,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:02,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295882481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:02,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:02,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295882481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:02,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-22T17:17:02,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:02,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/da9ef9fba2ba4042bc44b925897744ce 2024-11-22T17:17:02,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/2c9e1c58a13647dcb29258619bdbee85 is 50, key is test_row_0/C:col10/1732295821308/Put/seqid=0 2024-11-22T17:17:02,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742324_1500 (size=12001) 2024-11-22T17:17:02,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-22T17:17:02,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:02,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-22T17:17:02,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:02,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-22T17:17:02,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:02,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:02,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:03,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/2c9e1c58a13647dcb29258619bdbee85 2024-11-22T17:17:03,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/750954e81ac34bd195aeeafaa32cc160 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/750954e81ac34bd195aeeafaa32cc160 2024-11-22T17:17:03,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/750954e81ac34bd195aeeafaa32cc160, entries=150, sequenceid=16, filesize=30.2 K 2024-11-22T17:17:03,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/da9ef9fba2ba4042bc44b925897744ce as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/da9ef9fba2ba4042bc44b925897744ce 2024-11-22T17:17:03,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/da9ef9fba2ba4042bc44b925897744ce, entries=150, sequenceid=16, filesize=11.7 K 2024-11-22T17:17:03,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/2c9e1c58a13647dcb29258619bdbee85 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/2c9e1c58a13647dcb29258619bdbee85 2024-11-22T17:17:03,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/2c9e1c58a13647dcb29258619bdbee85, entries=150, sequenceid=16, filesize=11.7 K 2024-11-22T17:17:03,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 83841b437d25c9378a29982f12a6cd16 in 1769ms, sequenceid=16, compaction requested=false 2024-11-22T17:17:03,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:03,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-22T17:17:03,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:03,145 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T17:17:03,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:03,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:03,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:03,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:03,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:03,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:03,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220d8b9f3fc03b4722a12ae98c5ff33780_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295821338/Put/seqid=0 2024-11-22T17:17:03,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742325_1501 (size=12154) 2024-11-22T17:17:03,323 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T17:17:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-22T17:17:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:03,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:03,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295883495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:03,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295883497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295883498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:03,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295883501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:03,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295883501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:03,504 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,504 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:03,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:03,561 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220d8b9f3fc03b4722a12ae98c5ff33780_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220d8b9f3fc03b4722a12ae98c5ff33780_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:03,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/fe20332b098742c8847ed3edbac94be9, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:03,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/fe20332b098742c8847ed3edbac94be9 is 175, key is test_row_0/A:col10/1732295821338/Put/seqid=0 2024-11-22T17:17:03,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742326_1502 (size=30955) 2024-11-22T17:17:03,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295883601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:03,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295883612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:03,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295883612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:03,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295883811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295883821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:03,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295883822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:03,967 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/fe20332b098742c8847ed3edbac94be9 2024-11-22T17:17:03,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/daec5d2bf3bb41be84b3b15fe13c7e35 is 50, key is test_row_0/B:col10/1732295821338/Put/seqid=0 2024-11-22T17:17:03,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742327_1503 (size=12001) 2024-11-22T17:17:04,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295884121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:04,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295884131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:04,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295884131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:04,376 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/daec5d2bf3bb41be84b3b15fe13c7e35 2024-11-22T17:17:04,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/13e3cf4e79cc4f7bae0a1039b9f1ef08 is 50, key is test_row_0/C:col10/1732295821338/Put/seqid=0 2024-11-22T17:17:04,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742328_1504 (size=12001) 2024-11-22T17:17:04,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295884631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:04,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:04,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:04,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:04,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295884641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:04,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295884641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:04,785 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/13e3cf4e79cc4f7bae0a1039b9f1ef08 2024-11-22T17:17:04,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/fe20332b098742c8847ed3edbac94be9 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fe20332b098742c8847ed3edbac94be9 2024-11-22T17:17:04,792 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fe20332b098742c8847ed3edbac94be9, entries=150, sequenceid=36, filesize=30.2 K 2024-11-22T17:17:04,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/daec5d2bf3bb41be84b3b15fe13c7e35 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/daec5d2bf3bb41be84b3b15fe13c7e35 2024-11-22T17:17:04,795 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/daec5d2bf3bb41be84b3b15fe13c7e35, entries=150, sequenceid=36, filesize=11.7 K 2024-11-22T17:17:04,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/13e3cf4e79cc4f7bae0a1039b9f1ef08 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/13e3cf4e79cc4f7bae0a1039b9f1ef08 2024-11-22T17:17:04,799 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/13e3cf4e79cc4f7bae0a1039b9f1ef08, entries=150, sequenceid=36, filesize=11.7 K 2024-11-22T17:17:04,800 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 83841b437d25c9378a29982f12a6cd16 in 1655ms, sequenceid=36, compaction requested=false 2024-11-22T17:17:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-22T17:17:04,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-22T17:17:04,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-22T17:17:04,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4990 sec 2024-11-22T17:17:04,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 3.5100 sec 2024-11-22T17:17:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-22T17:17:05,472 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:17:05,473 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:17:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-22T17:17:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-22T17:17:05,475 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:17:05,475 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:17:05,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:17:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:05,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T17:17:05,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:05,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:05,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:05,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:05,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:05,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:05,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112250c2487443a34e31a1ce06fb8a93ab41_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295823496/Put/seqid=0 2024-11-22T17:17:05,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742329_1505 (size=14594) 2024-11-22T17:17:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-22T17:17:05,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295885580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295885581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:05,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:05,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-22T17:17:05,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:05,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:05,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:05,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:05,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295885651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:05,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295885651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:05,655 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:05,655 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:05,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295885661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:05,666 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:05,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295885691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295885692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:05,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-22T17:17:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:05,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-22T17:17:05,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295885901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:05,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295885902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:05,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-22T17:17:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:05,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:05,949 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:05,952 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112250c2487443a34e31a1ce06fb8a93ab41_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112250c2487443a34e31a1ce06fb8a93ab41_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:05,953 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/c445b2fdbc4343c89e8a57e64ad20c0b, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:05,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/c445b2fdbc4343c89e8a57e64ad20c0b is 175, key is test_row_0/A:col10/1732295823496/Put/seqid=0 2024-11-22T17:17:05,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742330_1506 (size=39549) 2024-11-22T17:17:05,958 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/c445b2fdbc4343c89e8a57e64ad20c0b 2024-11-22T17:17:05,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/333a9f0330ec41a9b9590594b57494fb is 50, key is test_row_0/B:col10/1732295823496/Put/seqid=0 2024-11-22T17:17:05,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742331_1507 (size=12001) 2024-11-22T17:17:06,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-22T17:17:06,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:06,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-22T17:17:06,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295886211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:06,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295886211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:06,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-22T17:17:06,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:06,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/333a9f0330ec41a9b9590594b57494fb 2024-11-22T17:17:06,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-22T17:17:06,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:06,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/7f7d44f1835d49f5a9a380ef98b8b73d is 50, key is test_row_0/C:col10/1732295823496/Put/seqid=0 2024-11-22T17:17:06,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742332_1508 (size=12001) 2024-11-22T17:17:06,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-22T17:17:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-22T17:17:06,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-22T17:17:06,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:06,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:06,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295886721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:06,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:06,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:06,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:06,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295886721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:06,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:06,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:06,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:06,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/7f7d44f1835d49f5a9a380ef98b8b73d 2024-11-22T17:17:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/c445b2fdbc4343c89e8a57e64ad20c0b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/c445b2fdbc4343c89e8a57e64ad20c0b 2024-11-22T17:17:06,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/c445b2fdbc4343c89e8a57e64ad20c0b, entries=200, sequenceid=49, filesize=38.6 K 2024-11-22T17:17:06,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/333a9f0330ec41a9b9590594b57494fb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/333a9f0330ec41a9b9590594b57494fb 2024-11-22T17:17:06,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/333a9f0330ec41a9b9590594b57494fb, entries=150, sequenceid=49, filesize=11.7 K 2024-11-22T17:17:06,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/7f7d44f1835d49f5a9a380ef98b8b73d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/7f7d44f1835d49f5a9a380ef98b8b73d 2024-11-22T17:17:06,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/7f7d44f1835d49f5a9a380ef98b8b73d, entries=150, sequenceid=49, filesize=11.7 K 2024-11-22T17:17:06,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 83841b437d25c9378a29982f12a6cd16 in 1314ms, sequenceid=49, compaction requested=true 2024-11-22T17:17:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:06,840 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:17:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:17:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:17:06,840 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:06,841 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:06,841 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/A is initiating minor compaction (all files) 2024-11-22T17:17:06,841 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/A in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,841 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/750954e81ac34bd195aeeafaa32cc160, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fe20332b098742c8847ed3edbac94be9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/c445b2fdbc4343c89e8a57e64ad20c0b] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=99.1 K 2024-11-22T17:17:06,841 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,841 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/750954e81ac34bd195aeeafaa32cc160, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fe20332b098742c8847ed3edbac94be9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/c445b2fdbc4343c89e8a57e64ad20c0b] 2024-11-22T17:17:06,850 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:06,850 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 750954e81ac34bd195aeeafaa32cc160, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732295821307 2024-11-22T17:17:06,850 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/B is initiating minor compaction (all files) 2024-11-22T17:17:06,850 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/B in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,850 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/da9ef9fba2ba4042bc44b925897744ce, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/daec5d2bf3bb41be84b3b15fe13c7e35, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/333a9f0330ec41a9b9590594b57494fb] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=35.2 K 2024-11-22T17:17:06,850 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe20332b098742c8847ed3edbac94be9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732295821333 2024-11-22T17:17:06,850 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting da9ef9fba2ba4042bc44b925897744ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732295821307 2024-11-22T17:17:06,850 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting c445b2fdbc4343c89e8a57e64ad20c0b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732295823494 2024-11-22T17:17:06,850 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting daec5d2bf3bb41be84b3b15fe13c7e35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732295821333 2024-11-22T17:17:06,851 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 333a9f0330ec41a9b9590594b57494fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732295823496 2024-11-22T17:17:06,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-22T17:17:06,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,855 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:06,855 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T17:17:06,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:06,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:06,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:06,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:06,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:06,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:06,857 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122c2706fec30b344b9a292ba2557908c9b_83841b437d25c9378a29982f12a6cd16 store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:06,857 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#B#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:06,858 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/75a5bbe84e66454a885de7899f20bb55 is 50, key is test_row_0/B:col10/1732295823496/Put/seqid=0 2024-11-22T17:17:06,859 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122c2706fec30b344b9a292ba2557908c9b_83841b437d25c9378a29982f12a6cd16, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:06,860 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c2706fec30b344b9a292ba2557908c9b_83841b437d25c9378a29982f12a6cd16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:06,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742334_1510 (size=4469) 2024-11-22T17:17:06,865 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#A#compaction#429 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:06,866 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/2ca9079dec874a80a4229dd78d50caf2 is 175, key is test_row_0/A:col10/1732295823496/Put/seqid=0 2024-11-22T17:17:06,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223ac254d1a402429bb359471cc0f3d5fb_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295825572/Put/seqid=0 2024-11-22T17:17:06,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742333_1509 (size=12104) 2024-11-22T17:17:06,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742335_1511 (size=31058) 2024-11-22T17:17:06,893 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/2ca9079dec874a80a4229dd78d50caf2 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/2ca9079dec874a80a4229dd78d50caf2 2024-11-22T17:17:06,897 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/A of 83841b437d25c9378a29982f12a6cd16 into 2ca9079dec874a80a4229dd78d50caf2(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:06,897 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:06,898 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/A, priority=13, startTime=1732295826840; duration=0sec 2024-11-22T17:17:06,898 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:06,898 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:A 2024-11-22T17:17:06,898 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:06,898 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:06,899 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/C is initiating minor compaction (all files) 2024-11-22T17:17:06,899 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/C in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:06,899 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/2c9e1c58a13647dcb29258619bdbee85, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/13e3cf4e79cc4f7bae0a1039b9f1ef08, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/7f7d44f1835d49f5a9a380ef98b8b73d] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=35.2 K 2024-11-22T17:17:06,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742336_1512 (size=12154) 2024-11-22T17:17:06,899 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c9e1c58a13647dcb29258619bdbee85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732295821307 2024-11-22T17:17:06,900 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13e3cf4e79cc4f7bae0a1039b9f1ef08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732295821333 2024-11-22T17:17:06,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:06,900 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f7d44f1835d49f5a9a380ef98b8b73d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732295823496 2024-11-22T17:17:06,904 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223ac254d1a402429bb359471cc0f3d5fb_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223ac254d1a402429bb359471cc0f3d5fb_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:06,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/37a8177d847243278409f64536ab9731, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:06,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/37a8177d847243278409f64536ab9731 is 175, key is test_row_0/A:col10/1732295825572/Put/seqid=0 2024-11-22T17:17:06,909 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#C#compaction#432 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:06,910 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/8242448fac4d403b83bb7878c3edd9cc is 50, key is test_row_0/C:col10/1732295823496/Put/seqid=0 2024-11-22T17:17:06,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742337_1513 (size=30955) 2024-11-22T17:17:06,918 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/37a8177d847243278409f64536ab9731 2024-11-22T17:17:06,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/b7b77d5cd63b484c8f1650a1e5ee7ca0 is 50, key is test_row_0/B:col10/1732295825572/Put/seqid=0 2024-11-22T17:17:06,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742338_1514 (size=12104) 2024-11-22T17:17:06,937 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/8242448fac4d403b83bb7878c3edd9cc as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/8242448fac4d403b83bb7878c3edd9cc 2024-11-22T17:17:06,941 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/C of 83841b437d25c9378a29982f12a6cd16 into 8242448fac4d403b83bb7878c3edd9cc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:06,941 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:06,941 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/C, priority=13, startTime=1732295826840; duration=0sec 2024-11-22T17:17:06,941 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:06,941 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:C 2024-11-22T17:17:06,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742339_1515 (size=12001) 2024-11-22T17:17:07,272 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/75a5bbe84e66454a885de7899f20bb55 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/75a5bbe84e66454a885de7899f20bb55 2024-11-22T17:17:07,276 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/B of 83841b437d25c9378a29982f12a6cd16 into 75a5bbe84e66454a885de7899f20bb55(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:07,276 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:07,276 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/B, priority=13, startTime=1732295826840; duration=0sec 2024-11-22T17:17:07,276 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:07,276 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:B 2024-11-22T17:17:07,346 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/b7b77d5cd63b484c8f1650a1e5ee7ca0 2024-11-22T17:17:07,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/f345e58ff9664f04aa9ddce126e300ee is 50, key is test_row_0/C:col10/1732295825572/Put/seqid=0 2024-11-22T17:17:07,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742340_1516 (size=12001) 2024-11-22T17:17:07,372 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/f345e58ff9664f04aa9ddce126e300ee 2024-11-22T17:17:07,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/37a8177d847243278409f64536ab9731 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/37a8177d847243278409f64536ab9731 2024-11-22T17:17:07,379 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/37a8177d847243278409f64536ab9731, entries=150, sequenceid=68, filesize=30.2 K 2024-11-22T17:17:07,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/b7b77d5cd63b484c8f1650a1e5ee7ca0 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/b7b77d5cd63b484c8f1650a1e5ee7ca0 2024-11-22T17:17:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,383 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/b7b77d5cd63b484c8f1650a1e5ee7ca0, entries=150, sequenceid=68, filesize=11.7 K 2024-11-22T17:17:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/f345e58ff9664f04aa9ddce126e300ee as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f345e58ff9664f04aa9ddce126e300ee 2024-11-22T17:17:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,387 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f345e58ff9664f04aa9ddce126e300ee, entries=150, sequenceid=68, filesize=11.7 K 2024-11-22T17:17:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,388 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 83841b437d25c9378a29982f12a6cd16 in 533ms, sequenceid=68, compaction requested=false 2024-11-22T17:17:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-22T17:17:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-22T17:17:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-22T17:17:07,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9140 sec 2024-11-22T17:17:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.9200 sec 2024-11-22T17:17:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-22T17:17:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,631 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:17:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,632 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:17:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-22T17:17:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,634 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:17:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-22T17:17:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,634 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:17:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:17:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:07,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:17:07,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:07,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:07,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:07,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:07,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:07,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:07,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122bc84c76ee90e47969009f97173ede769_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295827705/Put/seqid=0 2024-11-22T17:17:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742341_1517 (size=19474) 2024-11-22T17:17:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-22T17:17:07,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:07,753 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122bc84c76ee90e47969009f97173ede769_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bc84c76ee90e47969009f97173ede769_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:07,754 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/5cc58b6f02aa45419864181dabf463df, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:07,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/5cc58b6f02aa45419864181dabf463df is 175, key is test_row_0/A:col10/1732295827705/Put/seqid=0 2024-11-22T17:17:07,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732295827732, max timestamp=1732295827753Num uniques 859; 2024-11-22T17:17:07,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T17:17:07,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732295827732, max timestamp=1732295827753Num uniques 859; 2024-11-22T17:17:07,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T17:17:07,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732295827732, max timestamp=1732295827753Num uniques 859; 2024-11-22T17:17:07,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T17:17:07,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742342_1518 (size=56733) 2024-11-22T17:17:07,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:07,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:07,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295887773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:07,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295887774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:07,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-22T17:17:07,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:07,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:07,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:07,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:07,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:07,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:07,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:07,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295887781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:07,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295887784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:07,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295887784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:07,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:07,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295887892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295887892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:07,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295887892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:07,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295887901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:07,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:07,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295887901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:07,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:07,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:07,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-22T17:17:07,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:07,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:07,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:07,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:07,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:07,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-22T17:17:08,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-22T17:17:08,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:08,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:08,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:08,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295888101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295888101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295888101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295888111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295888111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,171 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/5cc58b6f02aa45419864181dabf463df 2024-11-22T17:17:08,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/9fe60281309a4f0591dba1d404bea7bb is 50, key is test_row_0/B:col10/1732295827705/Put/seqid=0 2024-11-22T17:17:08,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742343_1519 (size=12001) 2024-11-22T17:17:08,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-22T17:17:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:08,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-22T17:17:08,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-22T17:17:08,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:08,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:08,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:08,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295888411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295888411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295888411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295888421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295888421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-22T17:17:08,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:08,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:08,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:08,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/9fe60281309a4f0591dba1d404bea7bb 2024-11-22T17:17:08,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/f892b15243de4acaa3e4f730820c4cee is 50, key is test_row_0/C:col10/1732295827705/Put/seqid=0 2024-11-22T17:17:08,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742344_1520 (size=12001) 2024-11-22T17:17:08,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-22T17:17:08,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:08,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:08,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:08,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-22T17:17:08,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-22T17:17:08,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:08,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:08,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:08,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:08,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295888921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295888921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295888922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295888931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295888931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:08,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/f892b15243de4acaa3e4f730820c4cee 2024-11-22T17:17:08,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/5cc58b6f02aa45419864181dabf463df as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5cc58b6f02aa45419864181dabf463df 2024-11-22T17:17:09,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5cc58b6f02aa45419864181dabf463df, entries=300, sequenceid=81, filesize=55.4 K 2024-11-22T17:17:09,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/9fe60281309a4f0591dba1d404bea7bb as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/9fe60281309a4f0591dba1d404bea7bb 2024-11-22T17:17:09,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/9fe60281309a4f0591dba1d404bea7bb, entries=150, sequenceid=81, filesize=11.7 K 2024-11-22T17:17:09,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/f892b15243de4acaa3e4f730820c4cee as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f892b15243de4acaa3e4f730820c4cee 2024-11-22T17:17:09,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f892b15243de4acaa3e4f730820c4cee, entries=150, sequenceid=81, filesize=11.7 K 2024-11-22T17:17:09,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 83841b437d25c9378a29982f12a6cd16 in 1299ms, sequenceid=81, compaction requested=true 2024-11-22T17:17:09,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:09,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:17:09,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:09,007 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:09,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:17:09,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:09,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:17:09,007 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:09,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:09,008 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118746 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:09,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-22T17:17:09,008 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/A is initiating minor compaction (all files) 2024-11-22T17:17:09,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:09,008 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/A in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:09,008 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/2ca9079dec874a80a4229dd78d50caf2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/37a8177d847243278409f64536ab9731, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5cc58b6f02aa45419864181dabf463df] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=116.0 K 2024-11-22T17:17:09,008 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-22T17:17:09,008 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:09,008 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/2ca9079dec874a80a4229dd78d50caf2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/37a8177d847243278409f64536ab9731, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5cc58b6f02aa45419864181dabf463df] 2024-11-22T17:17:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:17:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:17:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:17:09,009 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:09,009 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/B is initiating minor compaction (all files) 2024-11-22T17:17:09,009 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/B in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:09,009 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/75a5bbe84e66454a885de7899f20bb55, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/b7b77d5cd63b484c8f1650a1e5ee7ca0, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/9fe60281309a4f0591dba1d404bea7bb] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=35.3 K 2024-11-22T17:17:09,010 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ca9079dec874a80a4229dd78d50caf2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732295823496 2024-11-22T17:17:09,010 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 75a5bbe84e66454a885de7899f20bb55, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732295823496 2024-11-22T17:17:09,010 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37a8177d847243278409f64536ab9731, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732295825560 2024-11-22T17:17:09,010 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cc58b6f02aa45419864181dabf463df, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732295827692 2024-11-22T17:17:09,010 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting b7b77d5cd63b484c8f1650a1e5ee7ca0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732295825560 2024-11-22T17:17:09,011 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fe60281309a4f0591dba1d404bea7bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732295827692 2024-11-22T17:17:09,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a5e0f6f13d1f44d2b8e1c5966b820a17_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295827779/Put/seqid=0 2024-11-22T17:17:09,021 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#B#compaction#439 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:09,021 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/017848d49c7d46e4a7e9a7ce8d6b75ee is 50, key is test_row_0/B:col10/1732295827705/Put/seqid=0 2024-11-22T17:17:09,022 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:09,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742345_1521 (size=12154) 2024-11-22T17:17:09,029 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122b86eef8ba6f14858ab6624ec23ad25b8_83841b437d25c9378a29982f12a6cd16 store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:09,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742346_1522 (size=12207) 2024-11-22T17:17:09,031 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122b86eef8ba6f14858ab6624ec23ad25b8_83841b437d25c9378a29982f12a6cd16, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:09,031 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b86eef8ba6f14858ab6624ec23ad25b8_83841b437d25c9378a29982f12a6cd16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:09,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742347_1523 (size=4469) 2024-11-22T17:17:09,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:09,426 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a5e0f6f13d1f44d2b8e1c5966b820a17_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a5e0f6f13d1f44d2b8e1c5966b820a17_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:09,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/60235508054742d29e32119856812116, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:09,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/60235508054742d29e32119856812116 is 175, key is test_row_0/A:col10/1732295827779/Put/seqid=0 2024-11-22T17:17:09,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742348_1524 (size=30955) 2024-11-22T17:17:09,440 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/017848d49c7d46e4a7e9a7ce8d6b75ee as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/017848d49c7d46e4a7e9a7ce8d6b75ee 2024-11-22T17:17:09,440 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#A#compaction#440 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:09,441 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/b03ca296f062480a8d53b7f958b51a7d is 175, key is test_row_0/A:col10/1732295827705/Put/seqid=0 2024-11-22T17:17:09,446 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/B of 83841b437d25c9378a29982f12a6cd16 into 017848d49c7d46e4a7e9a7ce8d6b75ee(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:09,447 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:09,447 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/B, priority=13, startTime=1732295829007; duration=0sec 2024-11-22T17:17:09,447 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:09,447 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:B 2024-11-22T17:17:09,447 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:09,448 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:09,448 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/C is initiating minor compaction (all files) 2024-11-22T17:17:09,448 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/C in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:09,448 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/8242448fac4d403b83bb7878c3edd9cc, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f345e58ff9664f04aa9ddce126e300ee, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f892b15243de4acaa3e4f730820c4cee] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=35.3 K 2024-11-22T17:17:09,448 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 8242448fac4d403b83bb7878c3edd9cc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732295823496 2024-11-22T17:17:09,449 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting f345e58ff9664f04aa9ddce126e300ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732295825560 2024-11-22T17:17:09,449 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting f892b15243de4acaa3e4f730820c4cee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732295827692 2024-11-22T17:17:09,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742349_1525 (size=31161) 2024-11-22T17:17:09,454 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/b03ca296f062480a8d53b7f958b51a7d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b03ca296f062480a8d53b7f958b51a7d 2024-11-22T17:17:09,457 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#C#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:09,458 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/3cccf4ad1cb94c3da5fca6a2da547211 is 50, key is test_row_0/C:col10/1732295827705/Put/seqid=0 2024-11-22T17:17:09,464 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/A of 83841b437d25c9378a29982f12a6cd16 into b03ca296f062480a8d53b7f958b51a7d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:09,464 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:09,464 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/A, priority=13, startTime=1732295829007; duration=0sec 2024-11-22T17:17:09,464 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:09,464 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:A 2024-11-22T17:17:09,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742350_1526 (size=12207) 2024-11-22T17:17:09,470 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/3cccf4ad1cb94c3da5fca6a2da547211 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3cccf4ad1cb94c3da5fca6a2da547211 2024-11-22T17:17:09,473 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/C of 83841b437d25c9378a29982f12a6cd16 into 3cccf4ad1cb94c3da5fca6a2da547211(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:09,473 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:09,474 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/C, priority=13, startTime=1732295829007; duration=0sec 2024-11-22T17:17:09,474 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:09,474 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:C 2024-11-22T17:17:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-22T17:17:09,837 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/60235508054742d29e32119856812116 2024-11-22T17:17:09,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/13549a72d8fd48b9a288741b55a1f755 is 50, key is test_row_0/B:col10/1732295827779/Put/seqid=0 2024-11-22T17:17:09,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742351_1527 (size=12001) 2024-11-22T17:17:09,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:09,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295889940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:09,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295889941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:09,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295889941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:09,946 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:09,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295889945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:09,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295889951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:09,954 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295890051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295890051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295890061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,248 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/13549a72d8fd48b9a288741b55a1f755 2024-11-22T17:17:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/3efdf7ef08cd4a698f7ac8dcc3b63c36 is 50, key is test_row_0/C:col10/1732295827779/Put/seqid=0 2024-11-22T17:17:10,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742352_1528 (size=12001) 2024-11-22T17:17:10,261 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/3efdf7ef08cd4a698f7ac8dcc3b63c36 2024-11-22T17:17:10,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/60235508054742d29e32119856812116 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/60235508054742d29e32119856812116 2024-11-22T17:17:10,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295890261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295890261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,270 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/60235508054742d29e32119856812116, entries=150, sequenceid=116, filesize=30.2 K 2024-11-22T17:17:10,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/13549a72d8fd48b9a288741b55a1f755 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/13549a72d8fd48b9a288741b55a1f755 2024-11-22T17:17:10,275 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/13549a72d8fd48b9a288741b55a1f755, entries=150, sequenceid=116, filesize=11.7 K 2024-11-22T17:17:10,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295890271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/3efdf7ef08cd4a698f7ac8dcc3b63c36 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3efdf7ef08cd4a698f7ac8dcc3b63c36 2024-11-22T17:17:10,279 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3efdf7ef08cd4a698f7ac8dcc3b63c36, entries=150, sequenceid=116, filesize=11.7 K 2024-11-22T17:17:10,280 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 83841b437d25c9378a29982f12a6cd16 in 1272ms, sequenceid=116, compaction requested=false 2024-11-22T17:17:10,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:10,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:10,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-22T17:17:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-22T17:17:10,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-22T17:17:10,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6470 sec 2024-11-22T17:17:10,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.6530 sec 2024-11-22T17:17:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:10,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T17:17:10,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:10,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:10,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:10,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:10,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:10,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:10,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d3098268c6d94c99baacd8c571f7ea1f_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295830573/Put/seqid=0 2024-11-22T17:17:10,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742353_1529 (size=17134) 2024-11-22T17:17:10,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295890604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295890605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295890608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295890721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295890721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295890721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295890932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295890932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295890932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:10,999 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:11,001 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d3098268c6d94c99baacd8c571f7ea1f_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d3098268c6d94c99baacd8c571f7ea1f_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:11,002 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/a721d6209e584a12bd1a68d8455089e5, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:11,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/a721d6209e584a12bd1a68d8455089e5 is 175, key is test_row_0/A:col10/1732295830573/Put/seqid=0 2024-11-22T17:17:11,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742354_1530 (size=48239) 2024-11-22T17:17:11,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295891241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295891241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:11,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:11,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295891241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:11,406 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/a721d6209e584a12bd1a68d8455089e5 2024-11-22T17:17:11,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/1adad94f7d6347fea069ee9bde06c6e7 is 50, key is test_row_0/B:col10/1732295830573/Put/seqid=0 2024-11-22T17:17:11,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742355_1531 (size=12101) 2024-11-22T17:17:11,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295891751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:11,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295891751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295891751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-22T17:17:11,801 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:17:11,801 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:17:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-22T17:17:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-22T17:17:11,803 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:17:11,804 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:17:11,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:17:11,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/1adad94f7d6347fea069ee9bde06c6e7 2024-11-22T17:17:11,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/bf003537c75443379c51bcca0f6e5c53 is 50, key is test_row_0/C:col10/1732295830573/Put/seqid=0 2024-11-22T17:17:11,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742356_1532 (size=12101) 2024-11-22T17:17:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-22T17:17:11,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-22T17:17:11,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:11,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:11,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295891961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:11,964 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:11,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295891971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:11,973 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:11,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:11,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:12,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-22T17:17:12,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:12,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:12,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:12,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:12,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-22T17:17:12,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/bf003537c75443379c51bcca0f6e5c53 2024-11-22T17:17:12,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/a721d6209e584a12bd1a68d8455089e5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a721d6209e584a12bd1a68d8455089e5 2024-11-22T17:17:12,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a721d6209e584a12bd1a68d8455089e5, entries=250, sequenceid=131, filesize=47.1 K 2024-11-22T17:17:12,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/1adad94f7d6347fea069ee9bde06c6e7 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/1adad94f7d6347fea069ee9bde06c6e7 2024-11-22T17:17:12,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/1adad94f7d6347fea069ee9bde06c6e7, entries=150, sequenceid=131, filesize=11.8 K 2024-11-22T17:17:12,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/bf003537c75443379c51bcca0f6e5c53 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/bf003537c75443379c51bcca0f6e5c53 2024-11-22T17:17:12,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/bf003537c75443379c51bcca0f6e5c53, entries=150, sequenceid=131, filesize=11.8 K 2024-11-22T17:17:12,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 83841b437d25c9378a29982f12a6cd16 in 1663ms, sequenceid=131, compaction requested=true 2024-11-22T17:17:12,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:12,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:17:12,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:12,238 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:12,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:17:12,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:12,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:17:12,238 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:12,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:12,238 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:12,238 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:12,238 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/A is initiating minor compaction (all files) 2024-11-22T17:17:12,238 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/B is initiating minor compaction (all files) 2024-11-22T17:17:12,239 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/A in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:12,239 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/B in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:12,239 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/017848d49c7d46e4a7e9a7ce8d6b75ee, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/13549a72d8fd48b9a288741b55a1f755, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/1adad94f7d6347fea069ee9bde06c6e7] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=35.5 K 2024-11-22T17:17:12,239 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b03ca296f062480a8d53b7f958b51a7d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/60235508054742d29e32119856812116, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a721d6209e584a12bd1a68d8455089e5] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=107.8 K 2024-11-22T17:17:12,239 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:12,239 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b03ca296f062480a8d53b7f958b51a7d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/60235508054742d29e32119856812116, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a721d6209e584a12bd1a68d8455089e5] 2024-11-22T17:17:12,239 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 017848d49c7d46e4a7e9a7ce8d6b75ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732295827692 2024-11-22T17:17:12,239 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting b03ca296f062480a8d53b7f958b51a7d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732295827692 2024-11-22T17:17:12,239 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 13549a72d8fd48b9a288741b55a1f755, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732295827768 2024-11-22T17:17:12,239 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60235508054742d29e32119856812116, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732295827768 2024-11-22T17:17:12,239 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 1adad94f7d6347fea069ee9bde06c6e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732295829944 2024-11-22T17:17:12,240 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting a721d6209e584a12bd1a68d8455089e5, keycount=250, bloomtype=ROW, size=47.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732295829938 2024-11-22T17:17:12,246 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#B#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:12,247 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/2d368d6158134635aad3393609eaa05c is 50, key is test_row_0/B:col10/1732295830573/Put/seqid=0 2024-11-22T17:17:12,248 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:12,257 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122e51f0c031c1640839b3f20f2a4a76aef_83841b437d25c9378a29982f12a6cd16 store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:12,259 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122e51f0c031c1640839b3f20f2a4a76aef_83841b437d25c9378a29982f12a6cd16, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:12,260 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e51f0c031c1640839b3f20f2a4a76aef_83841b437d25c9378a29982f12a6cd16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:12,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-22T17:17:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:12,265 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T17:17:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:12,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:12,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:12,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742357_1533 (size=12409) 2024-11-22T17:17:12,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742358_1534 (size=4469) 2024-11-22T17:17:12,271 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#A#compaction#448 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:12,272 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/b11a1ca818134dfba33d2bc2f2364843 is 175, key is test_row_0/A:col10/1732295830573/Put/seqid=0 2024-11-22T17:17:12,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e139d6463686472495cb5793fe5ddd2b_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295830606/Put/seqid=0 2024-11-22T17:17:12,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742359_1535 (size=31363) 2024-11-22T17:17:12,304 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/b11a1ca818134dfba33d2bc2f2364843 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b11a1ca818134dfba33d2bc2f2364843 2024-11-22T17:17:12,311 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/A of 83841b437d25c9378a29982f12a6cd16 into b11a1ca818134dfba33d2bc2f2364843(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:12,311 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:12,311 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/A, priority=13, startTime=1732295832238; duration=0sec 2024-11-22T17:17:12,311 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:12,311 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:A 2024-11-22T17:17:12,311 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:12,312 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:12,312 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/C is initiating minor compaction (all files) 2024-11-22T17:17:12,312 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/C in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:12,312 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3cccf4ad1cb94c3da5fca6a2da547211, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3efdf7ef08cd4a698f7ac8dcc3b63c36, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/bf003537c75443379c51bcca0f6e5c53] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=35.5 K 2024-11-22T17:17:12,313 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cccf4ad1cb94c3da5fca6a2da547211, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732295827692 2024-11-22T17:17:12,313 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3efdf7ef08cd4a698f7ac8dcc3b63c36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732295827768 2024-11-22T17:17:12,313 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf003537c75443379c51bcca0f6e5c53, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732295829944 2024-11-22T17:17:12,323 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#C#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:12,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742360_1536 (size=12304) 2024-11-22T17:17:12,323 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/51aa6e7c11df4584a9c3574fe36e2171 is 50, key is test_row_0/C:col10/1732295830573/Put/seqid=0 2024-11-22T17:17:12,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:12,337 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e139d6463686472495cb5793fe5ddd2b_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e139d6463686472495cb5793fe5ddd2b_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:12,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/9f8ec7ed452149d1ba3dc5f54297831b, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:12,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/9f8ec7ed452149d1ba3dc5f54297831b is 175, key is test_row_0/A:col10/1732295830606/Put/seqid=0 2024-11-22T17:17:12,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742361_1537 (size=12409) 2024-11-22T17:17:12,352 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/51aa6e7c11df4584a9c3574fe36e2171 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/51aa6e7c11df4584a9c3574fe36e2171 2024-11-22T17:17:12,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742362_1538 (size=31105) 2024-11-22T17:17:12,363 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/C of 83841b437d25c9378a29982f12a6cd16 into 51aa6e7c11df4584a9c3574fe36e2171(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:12,363 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:12,363 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/C, priority=13, startTime=1732295832238; duration=0sec 2024-11-22T17:17:12,363 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:12,363 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:C 2024-11-22T17:17:12,364 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/9f8ec7ed452149d1ba3dc5f54297831b 2024-11-22T17:17:12,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/c88508eb0bf74ddbaa3e563574b716e5 is 50, key is test_row_0/B:col10/1732295830606/Put/seqid=0 2024-11-22T17:17:12,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742363_1539 (size=12151) 2024-11-22T17:17:12,377 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/c88508eb0bf74ddbaa3e563574b716e5 2024-11-22T17:17:12,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/b9a881ed09674e4ca8c5d47a306dfc97 is 50, key is test_row_0/C:col10/1732295830606/Put/seqid=0 2024-11-22T17:17:12,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742364_1540 (size=12151) 2024-11-22T17:17:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-22T17:17:12,671 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/2d368d6158134635aad3393609eaa05c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2d368d6158134635aad3393609eaa05c 2024-11-22T17:17:12,675 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/B of 83841b437d25c9378a29982f12a6cd16 into 2d368d6158134635aad3393609eaa05c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:12,675 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:12,675 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/B, priority=13, startTime=1732295832238; duration=0sec 2024-11-22T17:17:12,675 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:12,675 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:B 2024-11-22T17:17:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:12,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:12,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:12,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295892773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:12,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:12,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:12,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:12,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:12,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295892775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:12,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:12,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295892776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:12,790 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/b9a881ed09674e4ca8c5d47a306dfc97 2024-11-22T17:17:12,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/9f8ec7ed452149d1ba3dc5f54297831b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9f8ec7ed452149d1ba3dc5f54297831b 2024-11-22T17:17:12,795 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9f8ec7ed452149d1ba3dc5f54297831b, entries=150, sequenceid=151, filesize=30.4 K 2024-11-22T17:17:12,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/c88508eb0bf74ddbaa3e563574b716e5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/c88508eb0bf74ddbaa3e563574b716e5 2024-11-22T17:17:12,798 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/c88508eb0bf74ddbaa3e563574b716e5, entries=150, sequenceid=151, filesize=11.9 K 2024-11-22T17:17:12,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/b9a881ed09674e4ca8c5d47a306dfc97 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/b9a881ed09674e4ca8c5d47a306dfc97 2024-11-22T17:17:12,803 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/b9a881ed09674e4ca8c5d47a306dfc97, entries=150, sequenceid=151, filesize=11.9 K 2024-11-22T17:17:12,804 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 83841b437d25c9378a29982f12a6cd16 in 539ms, sequenceid=151, compaction requested=false 2024-11-22T17:17:12,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:12,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:12,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-22T17:17:12,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-22T17:17:12,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-22T17:17:12,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0020 sec 2024-11-22T17:17:12,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.0060 sec 2024-11-22T17:17:12,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:12,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T17:17:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:12,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227edba3b0020f487e8b770c02ebac8572_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295832775/Put/seqid=0 2024-11-22T17:17:12,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742365_1541 (size=12304) 2024-11-22T17:17:12,916 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:12,920 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227edba3b0020f487e8b770c02ebac8572_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227edba3b0020f487e8b770c02ebac8572_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:12,921 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/66500cf735f94f36b419fe258e48fa39, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:12,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/66500cf735f94f36b419fe258e48fa39 is 175, key is test_row_0/A:col10/1732295832775/Put/seqid=0 2024-11-22T17:17:12,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295892915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:12,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295892916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:12,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295892917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:12,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742366_1542 (size=31105) 2024-11-22T17:17:12,936 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/66500cf735f94f36b419fe258e48fa39 2024-11-22T17:17:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-22T17:17:12,941 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:17:12,942 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:17:12,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/5d2de362e73346fdb0b820255af12467 is 50, key is test_row_0/B:col10/1732295832775/Put/seqid=0 2024-11-22T17:17:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-22T17:17:12,943 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:17:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T17:17:12,944 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:17:12,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:17:12,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742367_1543 (size=12151) 2024-11-22T17:17:13,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295893032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295893032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295893032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T17:17:13,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-22T17:17:13,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:13,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:13,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:13,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:13,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:13,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:13,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295893241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295893241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295893241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-22T17:17:13,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:13,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:13,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:13,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:13,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:13,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T17:17:13,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/5d2de362e73346fdb0b820255af12467 2024-11-22T17:17:13,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/994cf58a112e47738861692221b6c03d is 50, key is test_row_0/C:col10/1732295832775/Put/seqid=0 2024-11-22T17:17:13,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742368_1544 (size=12151) 2024-11-22T17:17:13,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/994cf58a112e47738861692221b6c03d 2024-11-22T17:17:13,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/66500cf735f94f36b419fe258e48fa39 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/66500cf735f94f36b419fe258e48fa39 2024-11-22T17:17:13,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/66500cf735f94f36b419fe258e48fa39, entries=150, sequenceid=167, filesize=30.4 K 2024-11-22T17:17:13,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/5d2de362e73346fdb0b820255af12467 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/5d2de362e73346fdb0b820255af12467 2024-11-22T17:17:13,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/5d2de362e73346fdb0b820255af12467, entries=150, sequenceid=167, filesize=11.9 K 2024-11-22T17:17:13,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/994cf58a112e47738861692221b6c03d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/994cf58a112e47738861692221b6c03d 2024-11-22T17:17:13,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/994cf58a112e47738861692221b6c03d, entries=150, sequenceid=167, filesize=11.9 K 2024-11-22T17:17:13,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 83841b437d25c9378a29982f12a6cd16 in 506ms, sequenceid=167, compaction requested=true 2024-11-22T17:17:13,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:13,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:17:13,389 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:13,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:13,389 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:13,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:17:13,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:13,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:17:13,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:13,390 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93573 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:13,390 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/A is initiating minor compaction (all files) 2024-11-22T17:17:13,390 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/A in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:13,390 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b11a1ca818134dfba33d2bc2f2364843, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9f8ec7ed452149d1ba3dc5f54297831b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/66500cf735f94f36b419fe258e48fa39] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=91.4 K 2024-11-22T17:17:13,391 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:13,391 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b11a1ca818134dfba33d2bc2f2364843, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9f8ec7ed452149d1ba3dc5f54297831b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/66500cf735f94f36b419fe258e48fa39] 2024-11-22T17:17:13,391 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:13,391 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/B is initiating minor compaction (all files) 2024-11-22T17:17:13,391 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/B in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:13,391 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2d368d6158134635aad3393609eaa05c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/c88508eb0bf74ddbaa3e563574b716e5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/5d2de362e73346fdb0b820255af12467] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=35.9 K 2024-11-22T17:17:13,391 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting b11a1ca818134dfba33d2bc2f2364843, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732295829944 2024-11-22T17:17:13,392 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d368d6158134635aad3393609eaa05c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732295829944 2024-11-22T17:17:13,392 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f8ec7ed452149d1ba3dc5f54297831b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732295830602 2024-11-22T17:17:13,392 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting c88508eb0bf74ddbaa3e563574b716e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732295830602 2024-11-22T17:17:13,392 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66500cf735f94f36b419fe258e48fa39, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732295832770 2024-11-22T17:17:13,392 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d2de362e73346fdb0b820255af12467, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732295832770 2024-11-22T17:17:13,397 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:13,399 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#B#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:13,399 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/ec4d9788f20d4af9addbbbf46d1e5744 is 50, key is test_row_0/B:col10/1732295832775/Put/seqid=0 2024-11-22T17:17:13,399 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122d2093c261570414a93aa6e7f93eb3df7_83841b437d25c9378a29982f12a6cd16 store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:13,401 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122d2093c261570414a93aa6e7f93eb3df7_83841b437d25c9378a29982f12a6cd16, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:13,401 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d2093c261570414a93aa6e7f93eb3df7_83841b437d25c9378a29982f12a6cd16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:13,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-22T17:17:13,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:13,402 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T17:17:13,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:13,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:13,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:13,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:13,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:13,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:13,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742370_1546 (size=4469) 2024-11-22T17:17:13,410 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#A#compaction#456 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:13,410 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/7443b4a53a6e40d19e56ddd3168abdf1 is 175, key is test_row_0/A:col10/1732295832775/Put/seqid=0 2024-11-22T17:17:13,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122fde12bdcec0f4f61ab9610f431978420_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295832915/Put/seqid=0 2024-11-22T17:17:13,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742369_1545 (size=12561) 2024-11-22T17:17:13,428 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/ec4d9788f20d4af9addbbbf46d1e5744 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/ec4d9788f20d4af9addbbbf46d1e5744 2024-11-22T17:17:13,433 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/B of 83841b437d25c9378a29982f12a6cd16 into ec4d9788f20d4af9addbbbf46d1e5744(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:13,433 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:13,433 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/B, priority=13, startTime=1732295833389; duration=0sec 2024-11-22T17:17:13,433 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:13,433 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:B 2024-11-22T17:17:13,433 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:13,434 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:13,434 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/C is initiating minor compaction (all files) 2024-11-22T17:17:13,434 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/C in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:13,435 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/51aa6e7c11df4584a9c3574fe36e2171, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/b9a881ed09674e4ca8c5d47a306dfc97, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/994cf58a112e47738861692221b6c03d] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=35.9 K 2024-11-22T17:17:13,435 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 51aa6e7c11df4584a9c3574fe36e2171, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732295829944 2024-11-22T17:17:13,435 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a881ed09674e4ca8c5d47a306dfc97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732295830602 2024-11-22T17:17:13,435 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 994cf58a112e47738861692221b6c03d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732295832770 2024-11-22T17:17:13,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742371_1547 (size=31515) 2024-11-22T17:17:13,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742372_1548 (size=12304) 2024-11-22T17:17:13,450 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#C#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:13,451 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/4741d43aedd944bd85e77ce9dfc9f4e8 is 50, key is test_row_0/C:col10/1732295832775/Put/seqid=0 2024-11-22T17:17:13,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742373_1549 (size=12561) 2024-11-22T17:17:13,462 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/4741d43aedd944bd85e77ce9dfc9f4e8 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/4741d43aedd944bd85e77ce9dfc9f4e8 2024-11-22T17:17:13,468 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/C of 83841b437d25c9378a29982f12a6cd16 into 4741d43aedd944bd85e77ce9dfc9f4e8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:13,468 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:13,468 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/C, priority=13, startTime=1732295833390; duration=0sec 2024-11-22T17:17:13,468 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:13,468 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:C 2024-11-22T17:17:13,516 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T17:17:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:13,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:13,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295893564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295893564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295893567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T17:17:13,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295893671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295893671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295893681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,846 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/7443b4a53a6e40d19e56ddd3168abdf1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/7443b4a53a6e40d19e56ddd3168abdf1 2024-11-22T17:17:13,849 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/A of 83841b437d25c9378a29982f12a6cd16 into 7443b4a53a6e40d19e56ddd3168abdf1(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:13,849 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:13,849 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/A, priority=13, startTime=1732295833389; duration=0sec 2024-11-22T17:17:13,849 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:13,849 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:A 2024-11-22T17:17:13,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:13,854 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122fde12bdcec0f4f61ab9610f431978420_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fde12bdcec0f4f61ab9610f431978420_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:13,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/4cc2bebdba804cbba29783957313fdff, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:13,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/4cc2bebdba804cbba29783957313fdff is 175, key is test_row_0/A:col10/1732295832915/Put/seqid=0 2024-11-22T17:17:13,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742374_1550 (size=31105) 2024-11-22T17:17:13,869 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/4cc2bebdba804cbba29783957313fdff 2024-11-22T17:17:13,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/bb0792f1398c43a3bf86ef6f0794a48c is 50, key is test_row_0/B:col10/1732295832915/Put/seqid=0 2024-11-22T17:17:13,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742375_1551 (size=12151) 2024-11-22T17:17:13,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295893881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295893881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:13,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295893891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T17:17:14,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295894191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:14,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295894191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:14,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295894201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:14,282 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/bb0792f1398c43a3bf86ef6f0794a48c 2024-11-22T17:17:14,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/5d07225a8d6b4bf3803c2103a08e84fd is 50, key is test_row_0/C:col10/1732295832915/Put/seqid=0 2024-11-22T17:17:14,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742376_1552 (size=12151) 2024-11-22T17:17:14,302 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/5d07225a8d6b4bf3803c2103a08e84fd 2024-11-22T17:17:14,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/4cc2bebdba804cbba29783957313fdff as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4cc2bebdba804cbba29783957313fdff 2024-11-22T17:17:14,314 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4cc2bebdba804cbba29783957313fdff, entries=150, sequenceid=186, filesize=30.4 K 2024-11-22T17:17:14,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/bb0792f1398c43a3bf86ef6f0794a48c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/bb0792f1398c43a3bf86ef6f0794a48c 2024-11-22T17:17:14,322 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/bb0792f1398c43a3bf86ef6f0794a48c, entries=150, sequenceid=186, filesize=11.9 K 2024-11-22T17:17:14,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/5d07225a8d6b4bf3803c2103a08e84fd as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/5d07225a8d6b4bf3803c2103a08e84fd 2024-11-22T17:17:14,327 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/5d07225a8d6b4bf3803c2103a08e84fd, entries=150, sequenceid=186, filesize=11.9 K 2024-11-22T17:17:14,332 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 83841b437d25c9378a29982f12a6cd16 in 930ms, sequenceid=186, compaction requested=false 2024-11-22T17:17:14,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:14,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:14,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-22T17:17:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-22T17:17:14,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-22T17:17:14,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3960 sec 2024-11-22T17:17:14,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.4030 sec 2024-11-22T17:17:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:14,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-22T17:17:14,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:14,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:14,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:14,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:14,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:14,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:14,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220cfe7b00c79d4042bd5bc792c370aadf_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295833553/Put/seqid=0 2024-11-22T17:17:14,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742377_1553 (size=17284) 2024-11-22T17:17:14,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295894732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:14,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295894734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:14,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295894735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:14,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295894841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:14,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295894841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:14,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295894851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:15,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295895051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:15,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295895051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:15,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295895061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T17:17:15,092 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:17:15,092 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:17:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-22T17:17:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T17:17:15,094 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:17:15,094 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:17:15,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:17:15,113 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:15,117 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220cfe7b00c79d4042bd5bc792c370aadf_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220cfe7b00c79d4042bd5bc792c370aadf_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:15,118 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/fa6db8de450c46939d81093ee3b5f570, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:15,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/fa6db8de450c46939d81093ee3b5f570 is 175, key is test_row_0/A:col10/1732295833553/Put/seqid=0 2024-11-22T17:17:15,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742378_1554 (size=48389) 2024-11-22T17:17:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T17:17:15,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:15,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:15,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295895361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:15,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:15,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295895362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:15,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:15,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295895371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:15,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:15,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:15,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:15,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:15,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T17:17:15,526 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/fa6db8de450c46939d81093ee3b5f570 2024-11-22T17:17:15,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/2291f918b3ed4f26a84630e189284be8 is 50, key is test_row_0/B:col10/1732295833553/Put/seqid=0 2024-11-22T17:17:15,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742379_1555 (size=12151) 2024-11-22T17:17:15,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:15,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:15,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:15,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:15,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:15,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:15,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:15,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:15,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T17:17:15,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:15,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:15,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:15,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:15,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:15,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295895872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295895872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:15,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295895881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:15,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/2291f918b3ed4f26a84630e189284be8 2024-11-22T17:17:15,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/6fb213a1bfd8494583a56ccc4d8ea75c is 50, key is test_row_0/C:col10/1732295833553/Put/seqid=0 2024-11-22T17:17:15,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742380_1556 (size=12151) 2024-11-22T17:17:15,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/6fb213a1bfd8494583a56ccc4d8ea75c 2024-11-22T17:17:15,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/fa6db8de450c46939d81093ee3b5f570 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fa6db8de450c46939d81093ee3b5f570 2024-11-22T17:17:15,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fa6db8de450c46939d81093ee3b5f570, entries=250, sequenceid=204, filesize=47.3 K 2024-11-22T17:17:15,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/2291f918b3ed4f26a84630e189284be8 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2291f918b3ed4f26a84630e189284be8 2024-11-22T17:17:15,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2291f918b3ed4f26a84630e189284be8, entries=150, sequenceid=204, filesize=11.9 K 2024-11-22T17:17:15,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/6fb213a1bfd8494583a56ccc4d8ea75c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/6fb213a1bfd8494583a56ccc4d8ea75c 2024-11-22T17:17:15,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/6fb213a1bfd8494583a56ccc4d8ea75c, entries=150, sequenceid=204, filesize=11.9 K 2024-11-22T17:17:15,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 83841b437d25c9378a29982f12a6cd16 in 1269ms, sequenceid=204, compaction requested=true 2024-11-22T17:17:15,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:15,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:17:15,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:15,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:17:15,974 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:15,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:15,974 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:15,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:17:15,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:15,974 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:15,974 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:15,974 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/A is initiating minor compaction (all files) 2024-11-22T17:17:15,974 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/B is initiating minor compaction (all files) 2024-11-22T17:17:15,974 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/A in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:15,974 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/B in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:15,974 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/7443b4a53a6e40d19e56ddd3168abdf1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4cc2bebdba804cbba29783957313fdff, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fa6db8de450c46939d81093ee3b5f570] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=108.4 K 2024-11-22T17:17:15,974 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/ec4d9788f20d4af9addbbbf46d1e5744, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/bb0792f1398c43a3bf86ef6f0794a48c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2291f918b3ed4f26a84630e189284be8] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=36.0 K 2024-11-22T17:17:15,974 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:15,974 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/7443b4a53a6e40d19e56ddd3168abdf1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4cc2bebdba804cbba29783957313fdff, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fa6db8de450c46939d81093ee3b5f570] 2024-11-22T17:17:15,975 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting ec4d9788f20d4af9addbbbf46d1e5744, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732295832770 2024-11-22T17:17:15,975 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7443b4a53a6e40d19e56ddd3168abdf1, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732295832770 2024-11-22T17:17:15,976 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting bb0792f1398c43a3bf86ef6f0794a48c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732295832900 2024-11-22T17:17:15,976 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cc2bebdba804cbba29783957313fdff, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732295832900 2024-11-22T17:17:15,978 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa6db8de450c46939d81093ee3b5f570, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732295833553 2024-11-22T17:17:15,978 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 2291f918b3ed4f26a84630e189284be8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732295833553 2024-11-22T17:17:15,986 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:15,987 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:15,988 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/6ce8ebc769d043cd9b4f1100385c6a7a is 50, key is test_row_0/B:col10/1732295833553/Put/seqid=0 2024-11-22T17:17:15,988 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122b3c27c91507a48fe99a651cf594876d3_83841b437d25c9378a29982f12a6cd16 store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:15,990 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122b3c27c91507a48fe99a651cf594876d3_83841b437d25c9378a29982f12a6cd16, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:15,990 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b3c27c91507a48fe99a651cf594876d3_83841b437d25c9378a29982f12a6cd16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:15,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742381_1557 (size=12663) 2024-11-22T17:17:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:16,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T17:17:16,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:16,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:16,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:16,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:16,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:16,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:16,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:16,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:16,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742382_1558 (size=4469) 2024-11-22T17:17:16,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223f3371a626c744e0b2638f84395c16bb_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295834733/Put/seqid=0 2024-11-22T17:17:16,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742383_1559 (size=14794) 2024-11-22T17:17:16,035 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:16,038 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223f3371a626c744e0b2638f84395c16bb_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223f3371a626c744e0b2638f84395c16bb_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:16,038 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/4db1c1bad90e4003bba45263592fd354, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:16,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/4db1c1bad90e4003bba45263592fd354 is 175, key is test_row_0/A:col10/1732295834733/Put/seqid=0 2024-11-22T17:17:16,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742384_1560 (size=39749) 2024-11-22T17:17:16,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295896040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:16,045 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/4db1c1bad90e4003bba45263592fd354 2024-11-22T17:17:16,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295896043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:16,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/db24c7af3d9a45e8b4ad9cbed3d7f170 is 50, key is test_row_0/B:col10/1732295834733/Put/seqid=0 2024-11-22T17:17:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:16,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742385_1561 (size=12151) 2024-11-22T17:17:16,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:16,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:16,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295896161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:16,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295896161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T17:17:16,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:16,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:16,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295896371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:16,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295896371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:16,400 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/6ce8ebc769d043cd9b4f1100385c6a7a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/6ce8ebc769d043cd9b4f1100385c6a7a 2024-11-22T17:17:16,404 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/B of 83841b437d25c9378a29982f12a6cd16 into 6ce8ebc769d043cd9b4f1100385c6a7a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:16,404 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:16,405 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/B, priority=13, startTime=1732295835974; duration=0sec 2024-11-22T17:17:16,405 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:16,405 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:B 2024-11-22T17:17:16,405 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:16,405 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:16,405 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/C is initiating minor compaction (all files) 2024-11-22T17:17:16,405 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/C in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,406 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/4741d43aedd944bd85e77ce9dfc9f4e8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/5d07225a8d6b4bf3803c2103a08e84fd, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/6fb213a1bfd8494583a56ccc4d8ea75c] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=36.0 K 2024-11-22T17:17:16,406 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 4741d43aedd944bd85e77ce9dfc9f4e8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732295832770 2024-11-22T17:17:16,406 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d07225a8d6b4bf3803c2103a08e84fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732295832900 2024-11-22T17:17:16,406 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fb213a1bfd8494583a56ccc4d8ea75c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732295833553 2024-11-22T17:17:16,410 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#A#compaction#466 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:16,411 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/a075ec8f48294d1bae01e68d03adef4f is 175, key is test_row_0/A:col10/1732295833553/Put/seqid=0 2024-11-22T17:17:16,412 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#C#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:16,413 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/4c1e95bf8aa64586ba9c75503c9cf2b5 is 50, key is test_row_0/C:col10/1732295833553/Put/seqid=0 2024-11-22T17:17:16,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742386_1562 (size=31617) 2024-11-22T17:17:16,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742387_1563 (size=12663) 2024-11-22T17:17:16,439 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/4c1e95bf8aa64586ba9c75503c9cf2b5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/4c1e95bf8aa64586ba9c75503c9cf2b5 2024-11-22T17:17:16,443 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/C of 83841b437d25c9378a29982f12a6cd16 into 4c1e95bf8aa64586ba9c75503c9cf2b5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:16,443 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:16,443 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/C, priority=13, startTime=1732295835974; duration=0sec 2024-11-22T17:17:16,443 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:16,443 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:C 2024-11-22T17:17:16,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/db24c7af3d9a45e8b4ad9cbed3d7f170 2024-11-22T17:17:16,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/b09f3d9d7be64003a1f8a494f06f4866 is 50, key is test_row_0/C:col10/1732295834733/Put/seqid=0 2024-11-22T17:17:16,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:16,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:16,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742388_1564 (size=12151) 2024-11-22T17:17:16,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:16,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:16,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295896681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295896681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:16,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:16,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:16,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,822 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/a075ec8f48294d1bae01e68d03adef4f as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a075ec8f48294d1bae01e68d03adef4f 2024-11-22T17:17:16,833 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/A of 83841b437d25c9378a29982f12a6cd16 into a075ec8f48294d1bae01e68d03adef4f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:16,833 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:16,833 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/A, priority=13, startTime=1732295835973; duration=0sec 2024-11-22T17:17:16,833 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:16,833 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:A 2024-11-22T17:17:16,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/b09f3d9d7be64003a1f8a494f06f4866 2024-11-22T17:17:16,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/4db1c1bad90e4003bba45263592fd354 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4db1c1bad90e4003bba45263592fd354 2024-11-22T17:17:16,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295896881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:16,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4db1c1bad90e4003bba45263592fd354, entries=200, sequenceid=222, filesize=38.8 K 2024-11-22T17:17:16,885 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:16,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:16,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/db24c7af3d9a45e8b4ad9cbed3d7f170 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/db24c7af3d9a45e8b4ad9cbed3d7f170 2024-11-22T17:17:16,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/db24c7af3d9a45e8b4ad9cbed3d7f170, entries=150, sequenceid=222, filesize=11.9 K 2024-11-22T17:17:16,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/b09f3d9d7be64003a1f8a494f06f4866 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/b09f3d9d7be64003a1f8a494f06f4866 2024-11-22T17:17:16,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/b09f3d9d7be64003a1f8a494f06f4866, entries=150, sequenceid=222, filesize=11.9 K 2024-11-22T17:17:16,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 83841b437d25c9378a29982f12a6cd16 in 891ms, sequenceid=222, compaction requested=false 2024-11-22T17:17:16,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:16,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-22T17:17:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:16,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a4d8fb7850204839ae5bf71dcecfb3fe_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295836034/Put/seqid=0 2024-11-22T17:17:16,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:16,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:16,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:16,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:16,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742389_1565 (size=14794) 2024-11-22T17:17:16,936 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:16,941 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a4d8fb7850204839ae5bf71dcecfb3fe_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a4d8fb7850204839ae5bf71dcecfb3fe_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:16,941 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/964cea3f29a54e5288e3174978e25e93, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:16,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/964cea3f29a54e5288e3174978e25e93 is 175, key is test_row_0/A:col10/1732295836034/Put/seqid=0 2024-11-22T17:17:16,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295896939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:16,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295896939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:16,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742390_1566 (size=39749) 2024-11-22T17:17:17,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295897052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295897052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:17,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:17,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:17,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295897191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:17,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295897192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:17,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:17,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:17,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T17:17:17,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295897261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295897261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:17,357 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/964cea3f29a54e5288e3174978e25e93 2024-11-22T17:17:17,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/2a62f5e322d143168d3c5c575aa06805 is 50, key is test_row_0/B:col10/1732295836034/Put/seqid=0 2024-11-22T17:17:17,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742391_1567 (size=12151) 2024-11-22T17:17:17,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:17,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:17,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:17,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:17,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295897571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:17,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295897572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:17,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:17,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:17,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/2a62f5e322d143168d3c5c575aa06805 2024-11-22T17:17:17,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/c8471584b5f84783b38ec7703f40b0b6 is 50, key is test_row_0/C:col10/1732295836034/Put/seqid=0 2024-11-22T17:17:17,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742392_1568 (size=12151) 2024-11-22T17:17:17,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:17,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:17,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:17,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:17,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:17,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:18,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295898081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:18,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:18,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295898091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:18,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:18,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:18,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:18,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:18,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/c8471584b5f84783b38ec7703f40b0b6 2024-11-22T17:17:18,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/964cea3f29a54e5288e3174978e25e93 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/964cea3f29a54e5288e3174978e25e93 2024-11-22T17:17:18,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/964cea3f29a54e5288e3174978e25e93, entries=200, sequenceid=241, filesize=38.8 K 2024-11-22T17:17:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/2a62f5e322d143168d3c5c575aa06805 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2a62f5e322d143168d3c5c575aa06805 2024-11-22T17:17:18,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2a62f5e322d143168d3c5c575aa06805, entries=150, sequenceid=241, filesize=11.9 K 2024-11-22T17:17:18,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/c8471584b5f84783b38ec7703f40b0b6 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c8471584b5f84783b38ec7703f40b0b6 2024-11-22T17:17:18,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c8471584b5f84783b38ec7703f40b0b6, entries=150, sequenceid=241, filesize=11.9 K 2024-11-22T17:17:18,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 83841b437d25c9378a29982f12a6cd16 in 1292ms, sequenceid=241, compaction requested=true 2024-11-22T17:17:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:17:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:18,189 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:18,189 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:17:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:17:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:18,190 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:18,190 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/B is initiating minor compaction (all files) 2024-11-22T17:17:18,190 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/B in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,190 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:18,190 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/A is initiating minor compaction (all files) 2024-11-22T17:17:18,190 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/A in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,191 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a075ec8f48294d1bae01e68d03adef4f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4db1c1bad90e4003bba45263592fd354, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/964cea3f29a54e5288e3174978e25e93] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=108.5 K 2024-11-22T17:17:18,191 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,191 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a075ec8f48294d1bae01e68d03adef4f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4db1c1bad90e4003bba45263592fd354, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/964cea3f29a54e5288e3174978e25e93] 2024-11-22T17:17:18,191 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/6ce8ebc769d043cd9b4f1100385c6a7a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/db24c7af3d9a45e8b4ad9cbed3d7f170, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2a62f5e322d143168d3c5c575aa06805] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=36.1 K 2024-11-22T17:17:18,191 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting a075ec8f48294d1bae01e68d03adef4f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732295833553 2024-11-22T17:17:18,191 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce8ebc769d043cd9b4f1100385c6a7a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732295833553 2024-11-22T17:17:18,192 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4db1c1bad90e4003bba45263592fd354, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732295834731 2024-11-22T17:17:18,192 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting db24c7af3d9a45e8b4ad9cbed3d7f170, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732295834732 2024-11-22T17:17:18,192 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 964cea3f29a54e5288e3174978e25e93, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732295836034 2024-11-22T17:17:18,192 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a62f5e322d143168d3c5c575aa06805, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732295836034 2024-11-22T17:17:18,199 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:18,199 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/6b59dbf5ef0f430d943d1e2f79b18264 is 50, key is test_row_0/B:col10/1732295836034/Put/seqid=0 2024-11-22T17:17:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:18,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-22T17:17:18,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:18,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:18,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:18,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:18,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:18,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:18,208 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:18,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742393_1569 (size=12765) 2024-11-22T17:17:18,217 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112236b07775bc3f4384bc30a3325fce9c90_83841b437d25c9378a29982f12a6cd16 store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:18,219 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112236b07775bc3f4384bc30a3325fce9c90_83841b437d25c9378a29982f12a6cd16, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:18,219 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112236b07775bc3f4384bc30a3325fce9c90_83841b437d25c9378a29982f12a6cd16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:18,221 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/6b59dbf5ef0f430d943d1e2f79b18264 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/6b59dbf5ef0f430d943d1e2f79b18264 2024-11-22T17:17:18,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220eaa77a5cc5f4afc957a5c50904d43a1_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295836936/Put/seqid=0 2024-11-22T17:17:18,228 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/B of 83841b437d25c9378a29982f12a6cd16 into 6b59dbf5ef0f430d943d1e2f79b18264(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:18,228 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:18,228 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/B, priority=13, startTime=1732295838189; duration=0sec 2024-11-22T17:17:18,228 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:18,228 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:B 2024-11-22T17:17:18,228 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:18,229 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:18,229 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/C is initiating minor compaction (all files) 2024-11-22T17:17:18,229 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/C in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,232 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/4c1e95bf8aa64586ba9c75503c9cf2b5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/b09f3d9d7be64003a1f8a494f06f4866, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c8471584b5f84783b38ec7703f40b0b6] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=36.1 K 2024-11-22T17:17:18,238 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c1e95bf8aa64586ba9c75503c9cf2b5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732295833553 2024-11-22T17:17:18,238 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting b09f3d9d7be64003a1f8a494f06f4866, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732295834732 2024-11-22T17:17:18,239 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting c8471584b5f84783b38ec7703f40b0b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732295836034 2024-11-22T17:17:18,246 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#C#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:18,246 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/a2151c7e0f12471498b97aa8e0afb591 is 50, key is test_row_0/C:col10/1732295836034/Put/seqid=0 2024-11-22T17:17:18,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742395_1571 (size=14844) 2024-11-22T17:17:18,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295898249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:18,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295898256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:18,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742394_1570 (size=4469) 2024-11-22T17:17:18,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742396_1572 (size=12765) 2024-11-22T17:17:18,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:18,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:18,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:18,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:18,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295898361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:18,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295898371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:18,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:18,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:18,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:18,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:18,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295898571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295898581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:18,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:18,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:18,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:18,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T17:17:18,657 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,660 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220eaa77a5cc5f4afc957a5c50904d43a1_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220eaa77a5cc5f4afc957a5c50904d43a1_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:18,661 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/673c29ac2ade46599334dcfa899672c5, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:18,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/673c29ac2ade46599334dcfa899672c5 is 175, key is test_row_0/A:col10/1732295836936/Put/seqid=0 2024-11-22T17:17:18,665 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#A#compaction#475 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:18,665 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/51924017bfbf40e99cb0537e1991cc79 is 175, key is test_row_0/A:col10/1732295836034/Put/seqid=0 2024-11-22T17:17:18,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742397_1573 (size=39799) 2024-11-22T17:17:18,671 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/673c29ac2ade46599334dcfa899672c5 2024-11-22T17:17:18,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742398_1574 (size=31719) 2024-11-22T17:17:18,686 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/51924017bfbf40e99cb0537e1991cc79 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/51924017bfbf40e99cb0537e1991cc79 2024-11-22T17:17:18,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/b3bc10a9aaaf44ef8baafbede76a137f is 50, key is test_row_0/B:col10/1732295836936/Put/seqid=0 2024-11-22T17:17:18,693 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/a2151c7e0f12471498b97aa8e0afb591 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/a2151c7e0f12471498b97aa8e0afb591 2024-11-22T17:17:18,701 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/A of 83841b437d25c9378a29982f12a6cd16 into 51924017bfbf40e99cb0537e1991cc79(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:18,701 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:18,701 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/A, priority=13, startTime=1732295838189; duration=0sec 2024-11-22T17:17:18,701 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:18,701 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:A 2024-11-22T17:17:18,705 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/C of 83841b437d25c9378a29982f12a6cd16 into a2151c7e0f12471498b97aa8e0afb591(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:18,705 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:18,705 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/C, priority=13, startTime=1732295838189; duration=0sec 2024-11-22T17:17:18,705 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:18,705 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:C 2024-11-22T17:17:18,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742399_1575 (size=12201) 2024-11-22T17:17:18,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/b3bc10a9aaaf44ef8baafbede76a137f 2024-11-22T17:17:18,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/bc3648cf75d148878eccf4b84866127d is 50, key is test_row_0/C:col10/1732295836936/Put/seqid=0 2024-11-22T17:17:18,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742400_1576 (size=12201) 2024-11-22T17:17:18,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/bc3648cf75d148878eccf4b84866127d 2024-11-22T17:17:18,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/673c29ac2ade46599334dcfa899672c5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/673c29ac2ade46599334dcfa899672c5 2024-11-22T17:17:18,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/673c29ac2ade46599334dcfa899672c5, entries=200, sequenceid=258, filesize=38.9 K 2024-11-22T17:17:18,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/b3bc10a9aaaf44ef8baafbede76a137f as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/b3bc10a9aaaf44ef8baafbede76a137f 2024-11-22T17:17:18,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/b3bc10a9aaaf44ef8baafbede76a137f, entries=150, sequenceid=258, filesize=11.9 K 2024-11-22T17:17:18,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/bc3648cf75d148878eccf4b84866127d as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/bc3648cf75d148878eccf4b84866127d 2024-11-22T17:17:18,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/bc3648cf75d148878eccf4b84866127d, entries=150, sequenceid=258, filesize=11.9 K 2024-11-22T17:17:18,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 83841b437d25c9378a29982f12a6cd16 in 545ms, sequenceid=258, compaction requested=false 2024-11-22T17:17:18,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:18,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T17:17:18,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,759 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-22T17:17:18,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:18,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:18,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:18,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:18,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:18,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:18,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223f5ffaf139cc445290f9f6216e41ea46_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295838245/Put/seqid=0 2024-11-22T17:17:18,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742401_1577 (size=12454) 2024-11-22T17:17:18,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,795 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223f5ffaf139cc445290f9f6216e41ea46_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223f5ffaf139cc445290f9f6216e41ea46_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:18,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/250220d5772046da996c23f6b709f623, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:18,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/250220d5772046da996c23f6b709f623 is 175, key is test_row_0/A:col10/1732295838245/Put/seqid=0 2024-11-22T17:17:18,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742402_1578 (size=31255) 2024-11-22T17:17:18,812 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/250220d5772046da996c23f6b709f623 2024-11-22T17:17:18,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/1f3f586758ed43b790af9a01a4dc4a45 is 50, key is test_row_0/B:col10/1732295838245/Put/seqid=0 2024-11-22T17:17:18,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742403_1579 (size=12301) 2024-11-22T17:17:18,827 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/1f3f586758ed43b790af9a01a4dc4a45 2024-11-22T17:17:18,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/793a8712d6de4845b1d560d01b98dfd3 is 50, key is test_row_0/C:col10/1732295838245/Put/seqid=0 2024-11-22T17:17:18,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742404_1580 (size=12301) 2024-11-22T17:17:18,867 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/793a8712d6de4845b1d560d01b98dfd3 2024-11-22T17:17:18,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/250220d5772046da996c23f6b709f623 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/250220d5772046da996c23f6b709f623 2024-11-22T17:17:18,884 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/250220d5772046da996c23f6b709f623, entries=150, sequenceid=276, filesize=30.5 K 2024-11-22T17:17:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:18,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/1f3f586758ed43b790af9a01a4dc4a45 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/1f3f586758ed43b790af9a01a4dc4a45 2024-11-22T17:17:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,889 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/1f3f586758ed43b790af9a01a4dc4a45, entries=150, sequenceid=276, filesize=12.0 K 2024-11-22T17:17:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/793a8712d6de4845b1d560d01b98dfd3 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/793a8712d6de4845b1d560d01b98dfd3 2024-11-22T17:17:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,898 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/793a8712d6de4845b1d560d01b98dfd3, entries=150, sequenceid=276, filesize=12.0 K 2024-11-22T17:17:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,899 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=13.42 KB/13740 for 83841b437d25c9378a29982f12a6cd16 in 140ms, sequenceid=276, compaction requested=true 2024-11-22T17:17:18,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:18,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:18,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-22T17:17:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-22T17:17:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-22T17:17:18,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8060 sec 2024-11-22T17:17:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.8110 sec 2024-11-22T17:17:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:18,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T17:17:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:18,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:18,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:18,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:18,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:18,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:18,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ce46211b41624c138cebaa67f375395d_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295838920/Put/seqid=0 2024-11-22T17:17:18,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742405_1581 (size=14994) 2024-11-22T17:17:18,954 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:18,957 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ce46211b41624c138cebaa67f375395d_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ce46211b41624c138cebaa67f375395d_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:18,958 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/9456bba36d1b4299a5142efe7fcd0ab4, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:18,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/9456bba36d1b4299a5142efe7fcd0ab4 is 175, key is test_row_0/A:col10/1732295838920/Put/seqid=0 2024-11-22T17:17:18,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742406_1582 (size=39949) 2024-11-22T17:17:18,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295898965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:18,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732295838955, max timestamp=1732295838970Num uniques 859; 2024-11-22T17:17:18,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T17:17:18,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295838955, max timestamp=1732295838970Num uniques 859; 2024-11-22T17:17:18,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:17:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:18,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732295838955, max timestamp=1732295838970Num uniques 859; 2024-11-22T17:17:18,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e61df7af61d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T17:17:19,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295899030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295899033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295899081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295899091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,097 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_0' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295899101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,106 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_1' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295899141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295899151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T17:17:19,272 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:17:19,272 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:17:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-22T17:17:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T17:17:19,274 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:17:19,274 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:17:19,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:17:19,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295899292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295899351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,362 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/9456bba36d1b4299a5142efe7fcd0ab4 2024-11-22T17:17:19,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295899361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/21a54507e2c54ef38c2084674bf1a6f5 is 50, key is test_row_0/B:col10/1732295838920/Put/seqid=0 2024-11-22T17:17:19,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742407_1583 (size=12301) 2024-11-22T17:17:19,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/21a54507e2c54ef38c2084674bf1a6f5 2024-11-22T17:17:19,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/ce5b6b6e4e8f4b9b962be151531960c1 is 50, key is test_row_0/C:col10/1732295838920/Put/seqid=0 2024-11-22T17:17:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T17:17:19,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742408_1584 (size=12301) 2024-11-22T17:17:19,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/ce5b6b6e4e8f4b9b962be151531960c1 2024-11-22T17:17:19,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/9456bba36d1b4299a5142efe7fcd0ab4 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9456bba36d1b4299a5142efe7fcd0ab4 2024-11-22T17:17:19,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9456bba36d1b4299a5142efe7fcd0ab4, entries=200, sequenceid=287, filesize=39.0 K 2024-11-22T17:17:19,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/21a54507e2c54ef38c2084674bf1a6f5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/21a54507e2c54ef38c2084674bf1a6f5 2024-11-22T17:17:19,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/21a54507e2c54ef38c2084674bf1a6f5, entries=150, sequenceid=287, filesize=12.0 K 2024-11-22T17:17:19,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/ce5b6b6e4e8f4b9b962be151531960c1 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/ce5b6b6e4e8f4b9b962be151531960c1 2024-11-22T17:17:19,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/ce5b6b6e4e8f4b9b962be151531960c1, entries=150, sequenceid=287, filesize=12.0 K 2024-11-22T17:17:19,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 83841b437d25c9378a29982f12a6cd16 in 474ms, sequenceid=287, compaction requested=true 2024-11-22T17:17:19,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:19,395 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:17:19,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:17:19,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:19,395 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:17:19,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:17:19,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:19,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:17:19,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:19,399 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142722 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:17:19,399 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/A is initiating minor compaction (all files) 2024-11-22T17:17:19,399 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/A in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:19,399 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/51924017bfbf40e99cb0537e1991cc79, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/673c29ac2ade46599334dcfa899672c5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/250220d5772046da996c23f6b709f623, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9456bba36d1b4299a5142efe7fcd0ab4] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=139.4 K 2024-11-22T17:17:19,399 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:19,399 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/51924017bfbf40e99cb0537e1991cc79, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/673c29ac2ade46599334dcfa899672c5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/250220d5772046da996c23f6b709f623, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9456bba36d1b4299a5142efe7fcd0ab4] 2024-11-22T17:17:19,399 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49568 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:17:19,399 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51924017bfbf40e99cb0537e1991cc79, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732295836034 2024-11-22T17:17:19,399 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/B is initiating minor compaction (all files) 2024-11-22T17:17:19,399 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/B in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:19,399 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/6b59dbf5ef0f430d943d1e2f79b18264, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/b3bc10a9aaaf44ef8baafbede76a137f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/1f3f586758ed43b790af9a01a4dc4a45, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/21a54507e2c54ef38c2084674bf1a6f5] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=48.4 K 2024-11-22T17:17:19,400 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 673c29ac2ade46599334dcfa899672c5, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732295836923 2024-11-22T17:17:19,400 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b59dbf5ef0f430d943d1e2f79b18264, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732295836034 2024-11-22T17:17:19,400 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting b3bc10a9aaaf44ef8baafbede76a137f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732295836923 2024-11-22T17:17:19,400 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 250220d5772046da996c23f6b709f623, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732295838239 2024-11-22T17:17:19,400 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3f586758ed43b790af9a01a4dc4a45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732295838239 2024-11-22T17:17:19,400 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9456bba36d1b4299a5142efe7fcd0ab4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732295838890 2024-11-22T17:17:19,401 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a54507e2c54ef38c2084674bf1a6f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732295838890 2024-11-22T17:17:19,407 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:19,408 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/a0d949ed91cb430b86bf756ebc9cb3e9 is 50, key is test_row_0/B:col10/1732295838920/Put/seqid=0 2024-11-22T17:17:19,408 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:19,410 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112272976826d58c4d67b0ea2f616607c443_83841b437d25c9378a29982f12a6cd16 store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:19,413 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112272976826d58c4d67b0ea2f616607c443_83841b437d25c9378a29982f12a6cd16, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:19,413 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112272976826d58c4d67b0ea2f616607c443_83841b437d25c9378a29982f12a6cd16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:19,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742409_1585 (size=13051) 2024-11-22T17:17:19,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742410_1586 (size=4469) 2024-11-22T17:17:19,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-22T17:17:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:19,426 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=129.71 KB heapSize=370.14 KB 2024-11-22T17:17:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:17:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:17:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:17:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227e028edcdd614e6c9e469b832c4b039b_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295839029/Put/seqid=0 2024-11-22T17:17:19,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742411_1587 (size=12454) 2024-11-22T17:17:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T17:17:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:19,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:19,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295899655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295899661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295899672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295899771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:19,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:19,821 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/a0d949ed91cb430b86bf756ebc9cb3e9 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/a0d949ed91cb430b86bf756ebc9cb3e9 2024-11-22T17:17:19,825 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/B of 83841b437d25c9378a29982f12a6cd16 into a0d949ed91cb430b86bf756ebc9cb3e9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:19,825 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:19,825 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/B, priority=12, startTime=1732295839395; duration=0sec 2024-11-22T17:17:19,825 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:19,825 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:B 2024-11-22T17:17:19,825 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T17:17:19,826 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49568 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T17:17:19,826 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:19,826 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/C is initiating minor compaction (all files) 2024-11-22T17:17:19,826 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/C in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:19,826 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/a2151c7e0f12471498b97aa8e0afb591, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/bc3648cf75d148878eccf4b84866127d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/793a8712d6de4845b1d560d01b98dfd3, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/ce5b6b6e4e8f4b9b962be151531960c1] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=48.4 K 2024-11-22T17:17:19,827 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/b7d7c2c3a775446683edc81b963b6c5b is 175, key is test_row_0/A:col10/1732295838920/Put/seqid=0 2024-11-22T17:17:19,827 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a2151c7e0f12471498b97aa8e0afb591, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732295836034 2024-11-22T17:17:19,827 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting bc3648cf75d148878eccf4b84866127d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732295836923 2024-11-22T17:17:19,827 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 793a8712d6de4845b1d560d01b98dfd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732295838239 2024-11-22T17:17:19,827 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting ce5b6b6e4e8f4b9b962be151531960c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732295838890 2024-11-22T17:17:19,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742412_1588 (size=32005) 2024-11-22T17:17:19,834 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/b7d7c2c3a775446683edc81b963b6c5b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b7d7c2c3a775446683edc81b963b6c5b 2024-11-22T17:17:19,836 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#C#compaction#489 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:19,837 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/a646b96b8c7b4d0da4fbca0ed4269381 is 50, key is test_row_0/C:col10/1732295838920/Put/seqid=0 2024-11-22T17:17:19,837 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/A of 83841b437d25c9378a29982f12a6cd16 into b7d7c2c3a775446683edc81b963b6c5b(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:19,837 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:19,837 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/A, priority=12, startTime=1732295839395; duration=0sec 2024-11-22T17:17:19,838 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:19,838 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:A 2024-11-22T17:17:19,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742413_1589 (size=13051) 2024-11-22T17:17:19,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:19,845 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227e028edcdd614e6c9e469b832c4b039b_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227e028edcdd614e6c9e469b832c4b039b_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/09124b0f56934bc3b8b194ea4726d566, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:19,846 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/a646b96b8c7b4d0da4fbca0ed4269381 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/a646b96b8c7b4d0da4fbca0ed4269381 2024-11-22T17:17:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/09124b0f56934bc3b8b194ea4726d566 is 175, key is test_row_0/A:col10/1732295839029/Put/seqid=0 2024-11-22T17:17:19,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742414_1590 (size=31255) 2024-11-22T17:17:19,854 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/09124b0f56934bc3b8b194ea4726d566 2024-11-22T17:17:19,862 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/C of 83841b437d25c9378a29982f12a6cd16 into a646b96b8c7b4d0da4fbca0ed4269381(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:19,862 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:19,862 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/C, priority=12, startTime=1732295839396; duration=0sec 2024-11-22T17:17:19,862 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:19,862 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:C 2024-11-22T17:17:19,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/cf130560ae264d3586197e4a8649495c is 50, key is test_row_0/B:col10/1732295839029/Put/seqid=0 2024-11-22T17:17:19,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742415_1591 (size=12301) 2024-11-22T17:17:19,870 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/cf130560ae264d3586197e4a8649495c 2024-11-22T17:17:19,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/9374d999d9d549fcbbe7cdc295468931 is 50, key is test_row_0/C:col10/1732295839029/Put/seqid=0 2024-11-22T17:17:19,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742416_1592 (size=12301) 2024-11-22T17:17:19,890 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/9374d999d9d549fcbbe7cdc295468931 2024-11-22T17:17:19,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/09124b0f56934bc3b8b194ea4726d566 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/09124b0f56934bc3b8b194ea4726d566 2024-11-22T17:17:19,897 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/09124b0f56934bc3b8b194ea4726d566, entries=150, sequenceid=323, filesize=30.5 K 2024-11-22T17:17:19,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/cf130560ae264d3586197e4a8649495c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/cf130560ae264d3586197e4a8649495c 2024-11-22T17:17:19,900 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/cf130560ae264d3586197e4a8649495c, entries=150, sequenceid=323, filesize=12.0 K 2024-11-22T17:17:19,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/9374d999d9d549fcbbe7cdc295468931 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/9374d999d9d549fcbbe7cdc295468931 2024-11-22T17:17:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T17:17:19,906 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/9374d999d9d549fcbbe7cdc295468931, entries=150, sequenceid=323, filesize=12.0 K 2024-11-22T17:17:19,907 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~370.09 KB/378976, currentSize=53.67 KB/54960 for 83841b437d25c9378a29982f12a6cd16 in 480ms, sequenceid=323, compaction requested=false 2024-11-22T17:17:19,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:19,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:19,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-22T17:17:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-22T17:17:19,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-22T17:17:19,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 634 msec 2024-11-22T17:17:19,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 637 msec 2024-11-22T17:17:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:19,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T17:17:19,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:19,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:19,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:19,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:19,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:19,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:19,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122bdb442c650c44c80ab860d9f77538d84_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295839982/Put/seqid=0 2024-11-22T17:17:19,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742417_1593 (size=14994) 2024-11-22T17:17:19,995 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:19,999 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122bdb442c650c44c80ab860d9f77538d84_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bdb442c650c44c80ab860d9f77538d84_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:20,000 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/7ebc31ac3a9d4b3d937663adab8bde2c, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:20,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/7ebc31ac3a9d4b3d937663adab8bde2c is 175, key is test_row_0/A:col10/1732295839982/Put/seqid=0 2024-11-22T17:17:20,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742418_1594 (size=39949) 2024-11-22T17:17:20,004 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/7ebc31ac3a9d4b3d937663adab8bde2c 2024-11-22T17:17:20,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/676b19354fe149519359dba2cd955bd3 is 50, key is test_row_0/B:col10/1732295839982/Put/seqid=0 2024-11-22T17:17:20,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742419_1595 (size=12301) 2024-11-22T17:17:20,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/676b19354fe149519359dba2cd955bd3 2024-11-22T17:17:20,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/f1648f4f25b6477da20a053153872a38 is 50, key is test_row_0/C:col10/1732295839982/Put/seqid=0 2024-11-22T17:17:20,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742420_1596 (size=12301) 2024-11-22T17:17:20,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295900066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:20,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:20,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295900172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:20,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:20,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295900181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295900181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:20,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295900392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T17:17:20,411 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:17:20,411 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T17:17:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-22T17:17:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T17:17:20,413 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T17:17:20,413 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T17:17:20,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T17:17:20,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/f1648f4f25b6477da20a053153872a38 2024-11-22T17:17:20,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/7ebc31ac3a9d4b3d937663adab8bde2c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/7ebc31ac3a9d4b3d937663adab8bde2c 2024-11-22T17:17:20,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/7ebc31ac3a9d4b3d937663adab8bde2c, entries=200, sequenceid=338, filesize=39.0 K 2024-11-22T17:17:20,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/676b19354fe149519359dba2cd955bd3 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/676b19354fe149519359dba2cd955bd3 2024-11-22T17:17:20,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/676b19354fe149519359dba2cd955bd3, entries=150, sequenceid=338, filesize=12.0 K 2024-11-22T17:17:20,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/f1648f4f25b6477da20a053153872a38 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f1648f4f25b6477da20a053153872a38 2024-11-22T17:17:20,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f1648f4f25b6477da20a053153872a38, entries=150, sequenceid=338, filesize=12.0 K 2024-11-22T17:17:20,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 83841b437d25c9378a29982f12a6cd16 in 454ms, sequenceid=338, compaction requested=true 2024-11-22T17:17:20,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:20,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:17:20,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:20,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:17:20,437 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:20,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:20,437 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:20,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:17:20,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:20,438 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:20,438 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/B is initiating minor compaction (all files) 2024-11-22T17:17:20,438 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/B in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:20,438 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/a0d949ed91cb430b86bf756ebc9cb3e9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/cf130560ae264d3586197e4a8649495c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/676b19354fe149519359dba2cd955bd3] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=36.8 K 2024-11-22T17:17:20,438 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:20,438 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/A is initiating minor compaction (all files) 2024-11-22T17:17:20,438 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/A in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:20,438 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b7d7c2c3a775446683edc81b963b6c5b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/09124b0f56934bc3b8b194ea4726d566, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/7ebc31ac3a9d4b3d937663adab8bde2c] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=100.8 K 2024-11-22T17:17:20,438 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:20,438 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b7d7c2c3a775446683edc81b963b6c5b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/09124b0f56934bc3b8b194ea4726d566, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/7ebc31ac3a9d4b3d937663adab8bde2c] 2024-11-22T17:17:20,439 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a0d949ed91cb430b86bf756ebc9cb3e9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732295838890 2024-11-22T17:17:20,439 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7d7c2c3a775446683edc81b963b6c5b, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732295838890 2024-11-22T17:17:20,439 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting cf130560ae264d3586197e4a8649495c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732295839021 2024-11-22T17:17:20,439 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09124b0f56934bc3b8b194ea4726d566, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732295839021 2024-11-22T17:17:20,439 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 676b19354fe149519359dba2cd955bd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732295839644 2024-11-22T17:17:20,439 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ebc31ac3a9d4b3d937663adab8bde2c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732295839607 2024-11-22T17:17:20,444 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:20,445 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122b5236a28d8d148f895a71e6bbb7085a8_83841b437d25c9378a29982f12a6cd16 store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:20,446 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#B#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:20,446 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/7d68ca54195441d4a3f40c1f26533543 is 50, key is test_row_0/B:col10/1732295839982/Put/seqid=0 2024-11-22T17:17:20,447 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122b5236a28d8d148f895a71e6bbb7085a8_83841b437d25c9378a29982f12a6cd16, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:20,448 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b5236a28d8d148f895a71e6bbb7085a8_83841b437d25c9378a29982f12a6cd16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:20,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742421_1597 (size=4469) 2024-11-22T17:17:20,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742422_1598 (size=13153) 2024-11-22T17:17:20,472 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/7d68ca54195441d4a3f40c1f26533543 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/7d68ca54195441d4a3f40c1f26533543 2024-11-22T17:17:20,478 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/B of 83841b437d25c9378a29982f12a6cd16 into 7d68ca54195441d4a3f40c1f26533543(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:20,478 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:20,478 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/B, priority=13, startTime=1732295840437; duration=0sec 2024-11-22T17:17:20,478 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:20,478 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:B 2024-11-22T17:17:20,478 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:20,479 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:20,479 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/C is initiating minor compaction (all files) 2024-11-22T17:17:20,479 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/C in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:20,479 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/a646b96b8c7b4d0da4fbca0ed4269381, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/9374d999d9d549fcbbe7cdc295468931, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f1648f4f25b6477da20a053153872a38] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=36.8 K 2024-11-22T17:17:20,479 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting a646b96b8c7b4d0da4fbca0ed4269381, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732295838890 2024-11-22T17:17:20,480 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 9374d999d9d549fcbbe7cdc295468931, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732295839021 2024-11-22T17:17:20,480 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting f1648f4f25b6477da20a053153872a38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732295839644 2024-11-22T17:17:20,490 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:20,490 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/c5ab81e40903428ca7af0b94844eb302 is 50, key is test_row_0/C:col10/1732295839982/Put/seqid=0 2024-11-22T17:17:20,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742423_1599 (size=13153) 2024-11-22T17:17:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T17:17:20,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42253 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-22T17:17:20,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:20,565 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T17:17:20,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:20,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:20,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:20,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:20,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:20,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:20,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112272b3739a74824d97905dba9ae95af1f0_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295840058/Put/seqid=0 2024-11-22T17:17:20,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742424_1600 (size=12454) 2024-11-22T17:17:20,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:20,601 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112272b3739a74824d97905dba9ae95af1f0_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112272b3739a74824d97905dba9ae95af1f0_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:20,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/5d826fe8ac9d4da4a5f513d18f8dfbbd, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:20,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/5d826fe8ac9d4da4a5f513d18f8dfbbd is 175, key is test_row_0/A:col10/1732295840058/Put/seqid=0 2024-11-22T17:17:20,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742425_1601 (size=31255) 2024-11-22T17:17:20,622 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/5d826fe8ac9d4da4a5f513d18f8dfbbd 2024-11-22T17:17:20,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/c04d12d2cc414cfebb5d32ea1eeb6248 is 50, key is test_row_0/B:col10/1732295840058/Put/seqid=0 2024-11-22T17:17:20,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742426_1602 (size=12301) 2024-11-22T17:17:20,643 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/c04d12d2cc414cfebb5d32ea1eeb6248 2024-11-22T17:17:20,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/05ba312a9cae483aa268535cf5025127 is 50, key is test_row_0/C:col10/1732295840058/Put/seqid=0 2024-11-22T17:17:20,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742427_1603 (size=12301) 2024-11-22T17:17:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:20,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. as already flushing 2024-11-22T17:17:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T17:17:20,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295900742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:20,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295900851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:20,858 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#A#compaction#495 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:20,858 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/be72dc43d3d54eff8a5b2b5d334a786c is 175, key is test_row_0/A:col10/1732295839982/Put/seqid=0 2024-11-22T17:17:20,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742428_1604 (size=32107) 2024-11-22T17:17:20,870 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/be72dc43d3d54eff8a5b2b5d334a786c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/be72dc43d3d54eff8a5b2b5d334a786c 2024-11-22T17:17:20,875 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/A of 83841b437d25c9378a29982f12a6cd16 into be72dc43d3d54eff8a5b2b5d334a786c(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:20,875 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:20,875 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/A, priority=13, startTime=1732295840437; duration=0sec 2024-11-22T17:17:20,876 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:20,876 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:A 2024-11-22T17:17:20,914 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/c5ab81e40903428ca7af0b94844eb302 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c5ab81e40903428ca7af0b94844eb302 2024-11-22T17:17:20,919 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/C of 83841b437d25c9378a29982f12a6cd16 into c5ab81e40903428ca7af0b94844eb302(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:20,919 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:20,919 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/C, priority=13, startTime=1732295840437; duration=0sec 2024-11-22T17:17:20,919 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:20,919 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:C 2024-11-22T17:17:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T17:17:21,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295901061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:21,074 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/05ba312a9cae483aa268535cf5025127 2024-11-22T17:17:21,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/5d826fe8ac9d4da4a5f513d18f8dfbbd as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5d826fe8ac9d4da4a5f513d18f8dfbbd 2024-11-22T17:17:21,080 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5d826fe8ac9d4da4a5f513d18f8dfbbd, entries=150, sequenceid=359, filesize=30.5 K 2024-11-22T17:17:21,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/c04d12d2cc414cfebb5d32ea1eeb6248 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/c04d12d2cc414cfebb5d32ea1eeb6248 2024-11-22T17:17:21,084 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/c04d12d2cc414cfebb5d32ea1eeb6248, entries=150, sequenceid=359, filesize=12.0 K 2024-11-22T17:17:21,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/05ba312a9cae483aa268535cf5025127 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/05ba312a9cae483aa268535cf5025127 2024-11-22T17:17:21,087 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/05ba312a9cae483aa268535cf5025127, entries=150, sequenceid=359, filesize=12.0 K 2024-11-22T17:17:21,088 INFO [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 83841b437d25c9378a29982f12a6cd16 in 523ms, sequenceid=359, compaction requested=false 2024-11-22T17:17:21,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:21,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:21,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-22T17:17:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-22T17:17:21,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-22T17:17:21,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 676 msec 2024-11-22T17:17:21,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 681 msec 2024-11-22T17:17:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:21,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T17:17:21,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:21,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:21,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:21,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:21,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:21,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:21,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227d4a1a24b9294cdeb04db5e170610493_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295841112/Put/seqid=0 2024-11-22T17:17:21,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742429_1605 (size=14994) 2024-11-22T17:17:21,123 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:21,127 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227d4a1a24b9294cdeb04db5e170610493_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227d4a1a24b9294cdeb04db5e170610493_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:21,128 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/17ef614921ed4ae1bbce9a6229ebdc0a, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:21,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/17ef614921ed4ae1bbce9a6229ebdc0a is 175, key is test_row_0/A:col10/1732295841112/Put/seqid=0 2024-11-22T17:17:21,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742430_1606 (size=39949) 2024-11-22T17:17:21,138 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/17ef614921ed4ae1bbce9a6229ebdc0a 2024-11-22T17:17:21,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/ccf1745865db44b4a54989e8863aa371 is 50, key is test_row_0/B:col10/1732295841112/Put/seqid=0 2024-11-22T17:17:21,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742431_1607 (size=12301) 2024-11-22T17:17:21,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/ccf1745865db44b4a54989e8863aa371 2024-11-22T17:17:21,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/3f9f654701d748918e738664fa8134c5 is 50, key is test_row_0/C:col10/1732295841112/Put/seqid=0 2024-11-22T17:17:21,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742432_1608 (size=12301) 2024-11-22T17:17:21,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295901168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:21,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295901173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:21,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59730 deadline: 1732295901191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:21,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1732295901191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:21,195 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_2' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,195 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to e61df7af61d2,42253,1732295655623 for 'test_row_0' in 83841b437d25c9378a29982f12a6cd16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-22T17:17:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:21,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295901281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:21,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295901281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:21,296 INFO [Thread-2257 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2257. 2024-11-22T17:17:21,296 DEBUG [Thread-2257 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:17:21,296 DEBUG [Thread-2257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,297 DEBUG [Thread-2257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,297 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:17:21,298 INFO [Thread-2251 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2251. 2024-11-22T17:17:21,298 DEBUG [Thread-2251 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:17:21,298 DEBUG [Thread-2251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,298 INFO [Thread-2255 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2255. 2024-11-22T17:17:21,298 DEBUG [Thread-2251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,298 DEBUG [Thread-2255 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:17:21,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:17:21,298 DEBUG [Thread-2255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,298 DEBUG [Thread-2255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:17:21,301 INFO [Thread-2253 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2253. 2024-11-22T17:17:21,302 DEBUG [Thread-2253 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:17:21,302 DEBUG [Thread-2253 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,302 DEBUG [Thread-2253 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:17:21,303 INFO [Thread-2249 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2249. 2024-11-22T17:17:21,303 DEBUG [Thread-2249 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:17:21,303 DEBUG [Thread-2249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,303 DEBUG [Thread-2249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:17:21,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59768 deadline: 1732295901371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:21,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:21,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T17:17:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59782 deadline: 1732295901492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1732295901491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 , the old value is region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T17:17:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., hostname=e61df7af61d2,42253,1732295655623, seqNum=5 because the exception is null or not the one we care about 2024-11-22T17:17:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T17:17:21,552 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:17:21,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/3f9f654701d748918e738664fa8134c5 2024-11-22T17:17:21,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/17ef614921ed4ae1bbce9a6229ebdc0a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/17ef614921ed4ae1bbce9a6229ebdc0a 2024-11-22T17:17:21,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/17ef614921ed4ae1bbce9a6229ebdc0a, entries=200, sequenceid=374, filesize=39.0 K 2024-11-22T17:17:21,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/ccf1745865db44b4a54989e8863aa371 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/ccf1745865db44b4a54989e8863aa371 2024-11-22T17:17:21,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/ccf1745865db44b4a54989e8863aa371, entries=150, sequenceid=374, filesize=12.0 K 2024-11-22T17:17:21,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/3f9f654701d748918e738664fa8134c5 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3f9f654701d748918e738664fa8134c5 2024-11-22T17:17:21,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3f9f654701d748918e738664fa8134c5, entries=150, sequenceid=374, filesize=12.0 K 2024-11-22T17:17:21,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 83841b437d25c9378a29982f12a6cd16 in 464ms, sequenceid=374, compaction requested=true 2024-11-22T17:17:21,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:21,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T17:17:21,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:21,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T17:17:21,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:21,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83841b437d25c9378a29982f12a6cd16:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T17:17:21,579 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:21,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:21,579 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:21,579 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:21,579 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:21,579 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/B is initiating minor compaction (all files) 2024-11-22T17:17:21,579 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/A is initiating minor compaction (all files) 2024-11-22T17:17:21,579 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/A in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:21,579 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/B in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:21,580 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/7d68ca54195441d4a3f40c1f26533543, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/c04d12d2cc414cfebb5d32ea1eeb6248, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/ccf1745865db44b4a54989e8863aa371] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=36.9 K 2024-11-22T17:17:21,580 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/be72dc43d3d54eff8a5b2b5d334a786c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5d826fe8ac9d4da4a5f513d18f8dfbbd, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/17ef614921ed4ae1bbce9a6229ebdc0a] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=100.9 K 2024-11-22T17:17:21,580 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:21,580 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. files: [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/be72dc43d3d54eff8a5b2b5d334a786c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5d826fe8ac9d4da4a5f513d18f8dfbbd, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/17ef614921ed4ae1bbce9a6229ebdc0a] 2024-11-22T17:17:21,580 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d68ca54195441d4a3f40c1f26533543, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732295839644 2024-11-22T17:17:21,580 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting be72dc43d3d54eff8a5b2b5d334a786c, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732295839644 2024-11-22T17:17:21,580 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting c04d12d2cc414cfebb5d32ea1eeb6248, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732295840038 2024-11-22T17:17:21,580 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d826fe8ac9d4da4a5f513d18f8dfbbd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732295840038 2024-11-22T17:17:21,580 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting ccf1745865db44b4a54989e8863aa371, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732295840736 2024-11-22T17:17:21,580 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17ef614921ed4ae1bbce9a6229ebdc0a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732295840731 2024-11-22T17:17:21,588 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:21,589 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#B#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:21,589 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/edb7dc21793746788fbaf72fa58c05dd is 50, key is test_row_0/B:col10/1732295841112/Put/seqid=0 2024-11-22T17:17:21,590 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122066ccace0c28401788daafd9e14abf4a_83841b437d25c9378a29982f12a6cd16 store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:21,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742433_1609 (size=13255) 2024-11-22T17:17:21,596 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122066ccace0c28401788daafd9e14abf4a_83841b437d25c9378a29982f12a6cd16, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:21,596 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122066ccace0c28401788daafd9e14abf4a_83841b437d25c9378a29982f12a6cd16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:21,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742434_1610 (size=4469) 2024-11-22T17:17:21,599 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#A#compaction#505 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:21,600 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/edb7dc21793746788fbaf72fa58c05dd as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/edb7dc21793746788fbaf72fa58c05dd 2024-11-22T17:17:21,600 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/3a8733fa59f84e4bbbff1f262ffd250b is 175, key is test_row_0/A:col10/1732295841112/Put/seqid=0 2024-11-22T17:17:21,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742435_1611 (size=32209) 2024-11-22T17:17:21,604 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/B of 83841b437d25c9378a29982f12a6cd16 into edb7dc21793746788fbaf72fa58c05dd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:21,604 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:21,604 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/B, priority=13, startTime=1732295841578; duration=0sec 2024-11-22T17:17:21,604 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T17:17:21,604 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:B 2024-11-22T17:17:21,604 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T17:17:21,604 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T17:17:21,604 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1541): 83841b437d25c9378a29982f12a6cd16/C is initiating minor compaction (all files) 2024-11-22T17:17:21,604 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83841b437d25c9378a29982f12a6cd16/C in TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:21,605 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c5ab81e40903428ca7af0b94844eb302, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/05ba312a9cae483aa268535cf5025127, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3f9f654701d748918e738664fa8134c5] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp, totalSize=36.9 K 2024-11-22T17:17:21,605 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting c5ab81e40903428ca7af0b94844eb302, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732295839644 2024-11-22T17:17:21,605 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ba312a9cae483aa268535cf5025127, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732295840038 2024-11-22T17:17:21,605 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f9f654701d748918e738664fa8134c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732295840736 2024-11-22T17:17:21,610 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83841b437d25c9378a29982f12a6cd16#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T17:17:21,610 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/7381c2104d6d4759a644fbde45a2d0a8 is 50, key is test_row_0/C:col10/1732295841112/Put/seqid=0 2024-11-22T17:17:21,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742436_1612 (size=13255) 2024-11-22T17:17:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42253 {}] regionserver.HRegion(8855): Flush requested on 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:21,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-22T17:17:21,803 INFO [Thread-2244 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2244. 2024-11-22T17:17:21,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:21,803 INFO [Thread-2240 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2240. 2024-11-22T17:17:21,803 DEBUG [Thread-2244 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:17:21,803 DEBUG [Thread-2240 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:17:21,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:17:21,803 DEBUG [Thread-2244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,803 DEBUG [Thread-2240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:21,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:17:21,803 DEBUG [Thread-2240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,803 DEBUG [Thread-2244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:21,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:17:21,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T17:17:21,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:17:21,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112222b912b1ce8b4d3299f1dded4f1bccee_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295841801/Put/seqid=0 2024-11-22T17:17:21,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742437_1613 (size=12454) 2024-11-22T17:17:21,882 INFO [Thread-2242 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2242. 2024-11-22T17:17:21,882 DEBUG [Thread-2242 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:17:21,882 DEBUG [Thread-2242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,882 DEBUG [Thread-2242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:21,882 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:17:22,007 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/3a8733fa59f84e4bbbff1f262ffd250b as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/3a8733fa59f84e4bbbff1f262ffd250b 2024-11-22T17:17:22,010 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/A of 83841b437d25c9378a29982f12a6cd16 into 3a8733fa59f84e4bbbff1f262ffd250b(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:22,010 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:22,010 INFO [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/A, priority=13, startTime=1732295841578; duration=0sec 2024-11-22T17:17:22,010 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:22,010 DEBUG [RS:0;e61df7af61d2:42253-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:A 2024-11-22T17:17:22,017 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/7381c2104d6d4759a644fbde45a2d0a8 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/7381c2104d6d4759a644fbde45a2d0a8 2024-11-22T17:17:22,020 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83841b437d25c9378a29982f12a6cd16/C of 83841b437d25c9378a29982f12a6cd16 into 7381c2104d6d4759a644fbde45a2d0a8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T17:17:22,020 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:22,020 INFO [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., storeName=83841b437d25c9378a29982f12a6cd16/C, priority=13, startTime=1732295841579; duration=0sec 2024-11-22T17:17:22,020 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T17:17:22,020 DEBUG [RS:0;e61df7af61d2:42253-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83841b437d25c9378a29982f12a6cd16:C 2024-11-22T17:17:22,211 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:22,215 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112222b912b1ce8b4d3299f1dded4f1bccee_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112222b912b1ce8b4d3299f1dded4f1bccee_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:22,215 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/29564308669446ce9eb7d56f5986447c, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:22,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/29564308669446ce9eb7d56f5986447c is 175, key is test_row_0/A:col10/1732295841801/Put/seqid=0 2024-11-22T17:17:22,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742438_1614 (size=31255) 2024-11-22T17:17:22,619 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/29564308669446ce9eb7d56f5986447c 2024-11-22T17:17:22,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/f414652f2a4449b18100a4883a07b53a is 50, key is test_row_0/B:col10/1732295841801/Put/seqid=0 2024-11-22T17:17:22,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742439_1615 (size=12301) 2024-11-22T17:17:23,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/f414652f2a4449b18100a4883a07b53a 2024-11-22T17:17:23,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/c003ff907f464df28a5905f4bad94903 is 50, key is test_row_0/C:col10/1732295841801/Put/seqid=0 2024-11-22T17:17:23,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742440_1616 (size=12301) 2024-11-22T17:17:23,213 INFO [Thread-2246 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2246. 2024-11-22T17:17:23,213 INFO [Thread-2238 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2238. 2024-11-22T17:17:23,213 DEBUG [Thread-2246 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:17:23,213 DEBUG [Thread-2238 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T17:17:23,213 DEBUG [Thread-2238 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:23,213 DEBUG [Thread-2246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:23,213 DEBUG [Thread-2246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:23,213 DEBUG [Thread-2238 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:23,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:17:23,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:17:23,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T17:17:23,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-22T17:17:23,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-22T17:17:23,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-22T17:17:23,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-22T17:17:23,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-22T17:17:23,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T17:17:23,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3077 2024-11-22T17:17:23,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2970 2024-11-22T17:17:23,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2980 2024-11-22T17:17:23,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3076 2024-11-22T17:17:23,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2971 2024-11-22T17:17:23,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T17:17:23,214 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T17:17:23,214 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T17:17:23,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:23,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:23,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:17:23,215 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T17:17:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T17:17:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T17:17:23,218 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295843217"}]},"ts":"1732295843217"} 2024-11-22T17:17:23,219 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T17:17:23,219 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T17:17:23,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T17:17:23,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, UNASSIGN}] 2024-11-22T17:17:23,222 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, UNASSIGN 2024-11-22T17:17:23,222 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=83841b437d25c9378a29982f12a6cd16, regionState=CLOSING, regionLocation=e61df7af61d2,42253,1732295655623 2024-11-22T17:17:23,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, UNASSIGN because future has completed 2024-11-22T17:17:23,223 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T17:17:23,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623}] 2024-11-22T17:17:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T17:17:23,376 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:23,376 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T17:17:23,376 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing 83841b437d25c9378a29982f12a6cd16, disabling compactions & flushes 2024-11-22T17:17:23,376 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:23,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/c003ff907f464df28a5905f4bad94903 2024-11-22T17:17:23,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/29564308669446ce9eb7d56f5986447c as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/29564308669446ce9eb7d56f5986447c 2024-11-22T17:17:23,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/29564308669446ce9eb7d56f5986447c, entries=150, sequenceid=396, filesize=30.5 K 2024-11-22T17:17:23,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/f414652f2a4449b18100a4883a07b53a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/f414652f2a4449b18100a4883a07b53a 2024-11-22T17:17:23,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/f414652f2a4449b18100a4883a07b53a, entries=150, sequenceid=396, filesize=12.0 K 2024-11-22T17:17:23,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/c003ff907f464df28a5905f4bad94903 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c003ff907f464df28a5905f4bad94903 2024-11-22T17:17:23,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c003ff907f464df28a5905f4bad94903, entries=150, sequenceid=396, filesize=12.0 K 2024-11-22T17:17:23,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=20.13 KB/20610 for 83841b437d25c9378a29982f12a6cd16 in 1649ms, sequenceid=396, compaction requested=false 2024-11-22T17:17:23,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83841b437d25c9378a29982f12a6cd16: 2024-11-22T17:17:23,451 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:23,451 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:23,451 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. after waiting 0 ms 2024-11-22T17:17:23,451 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:23,451 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing 83841b437d25c9378a29982f12a6cd16 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-22T17:17:23,451 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=A 2024-11-22T17:17:23,451 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:23,451 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=B 2024-11-22T17:17:23,451 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:23,451 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83841b437d25c9378a29982f12a6cd16, store=C 2024-11-22T17:17:23,451 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T17:17:23,455 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112293efde1b0fef423b84ae749c28ccb74a_83841b437d25c9378a29982f12a6cd16 is 50, key is test_row_0/A:col10/1732295843212/Put/seqid=0 2024-11-22T17:17:23,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742441_1617 (size=9914) 2024-11-22T17:17:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T17:17:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T17:17:23,858 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T17:17:23,861 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112293efde1b0fef423b84ae749c28ccb74a_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112293efde1b0fef423b84ae749c28ccb74a_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:23,862 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/a26345a11410456c8114e765549f256e, store: [table=TestAcidGuarantees family=A region=83841b437d25c9378a29982f12a6cd16] 2024-11-22T17:17:23,862 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/a26345a11410456c8114e765549f256e is 175, key is test_row_0/A:col10/1732295843212/Put/seqid=0 2024-11-22T17:17:23,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742442_1618 (size=22561) 2024-11-22T17:17:24,265 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/a26345a11410456c8114e765549f256e 2024-11-22T17:17:24,270 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/fc19c1db8cad42c994dcd0b1a777ddce is 50, key is test_row_0/B:col10/1732295843212/Put/seqid=0 2024-11-22T17:17:24,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742443_1619 (size=9857) 2024-11-22T17:17:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T17:17:24,673 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/fc19c1db8cad42c994dcd0b1a777ddce 2024-11-22T17:17:24,677 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/1c6c4967a35c491ba4e64dca9fde2d57 is 50, key is test_row_0/C:col10/1732295843212/Put/seqid=0 2024-11-22T17:17:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742444_1620 (size=9857) 2024-11-22T17:17:24,680 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/1c6c4967a35c491ba4e64dca9fde2d57 2024-11-22T17:17:24,683 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/A/a26345a11410456c8114e765549f256e as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a26345a11410456c8114e765549f256e 2024-11-22T17:17:24,686 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a26345a11410456c8114e765549f256e, entries=100, sequenceid=404, filesize=22.0 K 2024-11-22T17:17:24,686 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/B/fc19c1db8cad42c994dcd0b1a777ddce as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/fc19c1db8cad42c994dcd0b1a777ddce 2024-11-22T17:17:24,688 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/fc19c1db8cad42c994dcd0b1a777ddce, entries=100, sequenceid=404, filesize=9.6 K 2024-11-22T17:17:24,689 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/.tmp/C/1c6c4967a35c491ba4e64dca9fde2d57 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/1c6c4967a35c491ba4e64dca9fde2d57 2024-11-22T17:17:24,691 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/1c6c4967a35c491ba4e64dca9fde2d57, entries=100, sequenceid=404, filesize=9.6 K 2024-11-22T17:17:24,691 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 83841b437d25c9378a29982f12a6cd16 in 1240ms, sequenceid=404, compaction requested=true 2024-11-22T17:17:24,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/750954e81ac34bd195aeeafaa32cc160, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fe20332b098742c8847ed3edbac94be9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/c445b2fdbc4343c89e8a57e64ad20c0b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/2ca9079dec874a80a4229dd78d50caf2, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/37a8177d847243278409f64536ab9731, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5cc58b6f02aa45419864181dabf463df, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b03ca296f062480a8d53b7f958b51a7d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/60235508054742d29e32119856812116, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a721d6209e584a12bd1a68d8455089e5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b11a1ca818134dfba33d2bc2f2364843, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9f8ec7ed452149d1ba3dc5f54297831b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/7443b4a53a6e40d19e56ddd3168abdf1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/66500cf735f94f36b419fe258e48fa39, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4cc2bebdba804cbba29783957313fdff, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fa6db8de450c46939d81093ee3b5f570, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a075ec8f48294d1bae01e68d03adef4f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4db1c1bad90e4003bba45263592fd354, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/964cea3f29a54e5288e3174978e25e93, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/51924017bfbf40e99cb0537e1991cc79, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/673c29ac2ade46599334dcfa899672c5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/250220d5772046da996c23f6b709f623, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9456bba36d1b4299a5142efe7fcd0ab4, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b7d7c2c3a775446683edc81b963b6c5b, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/09124b0f56934bc3b8b194ea4726d566, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/7ebc31ac3a9d4b3d937663adab8bde2c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/be72dc43d3d54eff8a5b2b5d334a786c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5d826fe8ac9d4da4a5f513d18f8dfbbd, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/17ef614921ed4ae1bbce9a6229ebdc0a] to archive 2024-11-22T17:17:24,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:17:24,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/750954e81ac34bd195aeeafaa32cc160 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/750954e81ac34bd195aeeafaa32cc160 2024-11-22T17:17:24,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fe20332b098742c8847ed3edbac94be9 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fe20332b098742c8847ed3edbac94be9 2024-11-22T17:17:24,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/c445b2fdbc4343c89e8a57e64ad20c0b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/c445b2fdbc4343c89e8a57e64ad20c0b 2024-11-22T17:17:24,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/2ca9079dec874a80a4229dd78d50caf2 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/2ca9079dec874a80a4229dd78d50caf2 2024-11-22T17:17:24,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/37a8177d847243278409f64536ab9731 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/37a8177d847243278409f64536ab9731 2024-11-22T17:17:24,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5cc58b6f02aa45419864181dabf463df to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5cc58b6f02aa45419864181dabf463df 2024-11-22T17:17:24,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b03ca296f062480a8d53b7f958b51a7d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b03ca296f062480a8d53b7f958b51a7d 2024-11-22T17:17:24,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/60235508054742d29e32119856812116 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/60235508054742d29e32119856812116 2024-11-22T17:17:24,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a721d6209e584a12bd1a68d8455089e5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a721d6209e584a12bd1a68d8455089e5 2024-11-22T17:17:24,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b11a1ca818134dfba33d2bc2f2364843 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b11a1ca818134dfba33d2bc2f2364843 2024-11-22T17:17:24,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9f8ec7ed452149d1ba3dc5f54297831b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9f8ec7ed452149d1ba3dc5f54297831b 2024-11-22T17:17:24,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/7443b4a53a6e40d19e56ddd3168abdf1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/7443b4a53a6e40d19e56ddd3168abdf1 2024-11-22T17:17:24,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/66500cf735f94f36b419fe258e48fa39 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/66500cf735f94f36b419fe258e48fa39 2024-11-22T17:17:24,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4cc2bebdba804cbba29783957313fdff to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4cc2bebdba804cbba29783957313fdff 2024-11-22T17:17:24,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fa6db8de450c46939d81093ee3b5f570 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/fa6db8de450c46939d81093ee3b5f570 2024-11-22T17:17:24,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a075ec8f48294d1bae01e68d03adef4f to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a075ec8f48294d1bae01e68d03adef4f 2024-11-22T17:17:24,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4db1c1bad90e4003bba45263592fd354 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/4db1c1bad90e4003bba45263592fd354 2024-11-22T17:17:24,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/964cea3f29a54e5288e3174978e25e93 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/964cea3f29a54e5288e3174978e25e93 2024-11-22T17:17:24,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/51924017bfbf40e99cb0537e1991cc79 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/51924017bfbf40e99cb0537e1991cc79 2024-11-22T17:17:24,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/673c29ac2ade46599334dcfa899672c5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/673c29ac2ade46599334dcfa899672c5 2024-11-22T17:17:24,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/250220d5772046da996c23f6b709f623 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/250220d5772046da996c23f6b709f623 2024-11-22T17:17:24,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9456bba36d1b4299a5142efe7fcd0ab4 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/9456bba36d1b4299a5142efe7fcd0ab4 2024-11-22T17:17:24,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b7d7c2c3a775446683edc81b963b6c5b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/b7d7c2c3a775446683edc81b963b6c5b 2024-11-22T17:17:24,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/09124b0f56934bc3b8b194ea4726d566 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/09124b0f56934bc3b8b194ea4726d566 2024-11-22T17:17:24,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/7ebc31ac3a9d4b3d937663adab8bde2c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/7ebc31ac3a9d4b3d937663adab8bde2c 2024-11-22T17:17:24,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/be72dc43d3d54eff8a5b2b5d334a786c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/be72dc43d3d54eff8a5b2b5d334a786c 2024-11-22T17:17:24,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5d826fe8ac9d4da4a5f513d18f8dfbbd to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/5d826fe8ac9d4da4a5f513d18f8dfbbd 2024-11-22T17:17:24,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/17ef614921ed4ae1bbce9a6229ebdc0a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/17ef614921ed4ae1bbce9a6229ebdc0a 2024-11-22T17:17:24,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/da9ef9fba2ba4042bc44b925897744ce, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/daec5d2bf3bb41be84b3b15fe13c7e35, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/75a5bbe84e66454a885de7899f20bb55, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/333a9f0330ec41a9b9590594b57494fb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/b7b77d5cd63b484c8f1650a1e5ee7ca0, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/017848d49c7d46e4a7e9a7ce8d6b75ee, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/9fe60281309a4f0591dba1d404bea7bb, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/13549a72d8fd48b9a288741b55a1f755, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2d368d6158134635aad3393609eaa05c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/1adad94f7d6347fea069ee9bde06c6e7, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/c88508eb0bf74ddbaa3e563574b716e5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/ec4d9788f20d4af9addbbbf46d1e5744, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/5d2de362e73346fdb0b820255af12467, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/bb0792f1398c43a3bf86ef6f0794a48c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/6ce8ebc769d043cd9b4f1100385c6a7a, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2291f918b3ed4f26a84630e189284be8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/db24c7af3d9a45e8b4ad9cbed3d7f170, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/6b59dbf5ef0f430d943d1e2f79b18264, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2a62f5e322d143168d3c5c575aa06805, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/b3bc10a9aaaf44ef8baafbede76a137f, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/1f3f586758ed43b790af9a01a4dc4a45, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/a0d949ed91cb430b86bf756ebc9cb3e9, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/21a54507e2c54ef38c2084674bf1a6f5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/cf130560ae264d3586197e4a8649495c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/7d68ca54195441d4a3f40c1f26533543, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/676b19354fe149519359dba2cd955bd3, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/c04d12d2cc414cfebb5d32ea1eeb6248, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/ccf1745865db44b4a54989e8863aa371] to archive 2024-11-22T17:17:24,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:17:24,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/da9ef9fba2ba4042bc44b925897744ce to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/da9ef9fba2ba4042bc44b925897744ce 2024-11-22T17:17:24,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/daec5d2bf3bb41be84b3b15fe13c7e35 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/daec5d2bf3bb41be84b3b15fe13c7e35 2024-11-22T17:17:24,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/75a5bbe84e66454a885de7899f20bb55 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/75a5bbe84e66454a885de7899f20bb55 2024-11-22T17:17:24,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/333a9f0330ec41a9b9590594b57494fb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/333a9f0330ec41a9b9590594b57494fb 2024-11-22T17:17:24,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/b7b77d5cd63b484c8f1650a1e5ee7ca0 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/b7b77d5cd63b484c8f1650a1e5ee7ca0 2024-11-22T17:17:24,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/017848d49c7d46e4a7e9a7ce8d6b75ee to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/017848d49c7d46e4a7e9a7ce8d6b75ee 2024-11-22T17:17:24,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/9fe60281309a4f0591dba1d404bea7bb to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/9fe60281309a4f0591dba1d404bea7bb 2024-11-22T17:17:24,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/13549a72d8fd48b9a288741b55a1f755 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/13549a72d8fd48b9a288741b55a1f755 2024-11-22T17:17:24,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2d368d6158134635aad3393609eaa05c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2d368d6158134635aad3393609eaa05c 2024-11-22T17:17:24,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/1adad94f7d6347fea069ee9bde06c6e7 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/1adad94f7d6347fea069ee9bde06c6e7 2024-11-22T17:17:24,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/c88508eb0bf74ddbaa3e563574b716e5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/c88508eb0bf74ddbaa3e563574b716e5 2024-11-22T17:17:24,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/ec4d9788f20d4af9addbbbf46d1e5744 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/ec4d9788f20d4af9addbbbf46d1e5744 2024-11-22T17:17:24,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/5d2de362e73346fdb0b820255af12467 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/5d2de362e73346fdb0b820255af12467 2024-11-22T17:17:24,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/bb0792f1398c43a3bf86ef6f0794a48c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/bb0792f1398c43a3bf86ef6f0794a48c 2024-11-22T17:17:24,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/6ce8ebc769d043cd9b4f1100385c6a7a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/6ce8ebc769d043cd9b4f1100385c6a7a 2024-11-22T17:17:24,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2291f918b3ed4f26a84630e189284be8 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2291f918b3ed4f26a84630e189284be8 2024-11-22T17:17:24,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/db24c7af3d9a45e8b4ad9cbed3d7f170 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/db24c7af3d9a45e8b4ad9cbed3d7f170 2024-11-22T17:17:24,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/6b59dbf5ef0f430d943d1e2f79b18264 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/6b59dbf5ef0f430d943d1e2f79b18264 2024-11-22T17:17:24,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2a62f5e322d143168d3c5c575aa06805 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/2a62f5e322d143168d3c5c575aa06805 2024-11-22T17:17:24,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/b3bc10a9aaaf44ef8baafbede76a137f to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/b3bc10a9aaaf44ef8baafbede76a137f 2024-11-22T17:17:24,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/1f3f586758ed43b790af9a01a4dc4a45 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/1f3f586758ed43b790af9a01a4dc4a45 2024-11-22T17:17:24,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/a0d949ed91cb430b86bf756ebc9cb3e9 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/a0d949ed91cb430b86bf756ebc9cb3e9 2024-11-22T17:17:24,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/21a54507e2c54ef38c2084674bf1a6f5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/21a54507e2c54ef38c2084674bf1a6f5 2024-11-22T17:17:24,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/cf130560ae264d3586197e4a8649495c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/cf130560ae264d3586197e4a8649495c 2024-11-22T17:17:24,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/7d68ca54195441d4a3f40c1f26533543 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/7d68ca54195441d4a3f40c1f26533543 2024-11-22T17:17:24,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/676b19354fe149519359dba2cd955bd3 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/676b19354fe149519359dba2cd955bd3 2024-11-22T17:17:24,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/c04d12d2cc414cfebb5d32ea1eeb6248 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/c04d12d2cc414cfebb5d32ea1eeb6248 2024-11-22T17:17:24,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/ccf1745865db44b4a54989e8863aa371 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/ccf1745865db44b4a54989e8863aa371 2024-11-22T17:17:24,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/2c9e1c58a13647dcb29258619bdbee85, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/13e3cf4e79cc4f7bae0a1039b9f1ef08, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/8242448fac4d403b83bb7878c3edd9cc, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/7f7d44f1835d49f5a9a380ef98b8b73d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f345e58ff9664f04aa9ddce126e300ee, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3cccf4ad1cb94c3da5fca6a2da547211, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f892b15243de4acaa3e4f730820c4cee, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3efdf7ef08cd4a698f7ac8dcc3b63c36, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/51aa6e7c11df4584a9c3574fe36e2171, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/bf003537c75443379c51bcca0f6e5c53, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/b9a881ed09674e4ca8c5d47a306dfc97, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/4741d43aedd944bd85e77ce9dfc9f4e8, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/994cf58a112e47738861692221b6c03d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/5d07225a8d6b4bf3803c2103a08e84fd, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/4c1e95bf8aa64586ba9c75503c9cf2b5, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/6fb213a1bfd8494583a56ccc4d8ea75c, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/b09f3d9d7be64003a1f8a494f06f4866, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/a2151c7e0f12471498b97aa8e0afb591, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c8471584b5f84783b38ec7703f40b0b6, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/bc3648cf75d148878eccf4b84866127d, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/793a8712d6de4845b1d560d01b98dfd3, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/a646b96b8c7b4d0da4fbca0ed4269381, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/ce5b6b6e4e8f4b9b962be151531960c1, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/9374d999d9d549fcbbe7cdc295468931, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c5ab81e40903428ca7af0b94844eb302, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f1648f4f25b6477da20a053153872a38, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/05ba312a9cae483aa268535cf5025127, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3f9f654701d748918e738664fa8134c5] to archive 2024-11-22T17:17:24,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T17:17:24,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/2c9e1c58a13647dcb29258619bdbee85 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/2c9e1c58a13647dcb29258619bdbee85 2024-11-22T17:17:24,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/13e3cf4e79cc4f7bae0a1039b9f1ef08 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/13e3cf4e79cc4f7bae0a1039b9f1ef08 2024-11-22T17:17:24,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/8242448fac4d403b83bb7878c3edd9cc to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/8242448fac4d403b83bb7878c3edd9cc 2024-11-22T17:17:24,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/7f7d44f1835d49f5a9a380ef98b8b73d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/7f7d44f1835d49f5a9a380ef98b8b73d 2024-11-22T17:17:24,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f345e58ff9664f04aa9ddce126e300ee to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f345e58ff9664f04aa9ddce126e300ee 2024-11-22T17:17:24,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3cccf4ad1cb94c3da5fca6a2da547211 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3cccf4ad1cb94c3da5fca6a2da547211 2024-11-22T17:17:24,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f892b15243de4acaa3e4f730820c4cee to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f892b15243de4acaa3e4f730820c4cee 2024-11-22T17:17:24,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3efdf7ef08cd4a698f7ac8dcc3b63c36 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3efdf7ef08cd4a698f7ac8dcc3b63c36 2024-11-22T17:17:24,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/51aa6e7c11df4584a9c3574fe36e2171 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/51aa6e7c11df4584a9c3574fe36e2171 2024-11-22T17:17:24,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/bf003537c75443379c51bcca0f6e5c53 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/bf003537c75443379c51bcca0f6e5c53 2024-11-22T17:17:24,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/b9a881ed09674e4ca8c5d47a306dfc97 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/b9a881ed09674e4ca8c5d47a306dfc97 2024-11-22T17:17:24,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/4741d43aedd944bd85e77ce9dfc9f4e8 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/4741d43aedd944bd85e77ce9dfc9f4e8 2024-11-22T17:17:24,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/994cf58a112e47738861692221b6c03d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/994cf58a112e47738861692221b6c03d 2024-11-22T17:17:24,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/5d07225a8d6b4bf3803c2103a08e84fd to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/5d07225a8d6b4bf3803c2103a08e84fd 2024-11-22T17:17:24,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/4c1e95bf8aa64586ba9c75503c9cf2b5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/4c1e95bf8aa64586ba9c75503c9cf2b5 2024-11-22T17:17:24,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/6fb213a1bfd8494583a56ccc4d8ea75c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/6fb213a1bfd8494583a56ccc4d8ea75c 2024-11-22T17:17:24,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/b09f3d9d7be64003a1f8a494f06f4866 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/b09f3d9d7be64003a1f8a494f06f4866 2024-11-22T17:17:24,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/a2151c7e0f12471498b97aa8e0afb591 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/a2151c7e0f12471498b97aa8e0afb591 2024-11-22T17:17:24,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c8471584b5f84783b38ec7703f40b0b6 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c8471584b5f84783b38ec7703f40b0b6 2024-11-22T17:17:24,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/bc3648cf75d148878eccf4b84866127d to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/bc3648cf75d148878eccf4b84866127d 2024-11-22T17:17:24,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/793a8712d6de4845b1d560d01b98dfd3 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/793a8712d6de4845b1d560d01b98dfd3 2024-11-22T17:17:24,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/a646b96b8c7b4d0da4fbca0ed4269381 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/a646b96b8c7b4d0da4fbca0ed4269381 2024-11-22T17:17:24,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/ce5b6b6e4e8f4b9b962be151531960c1 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/ce5b6b6e4e8f4b9b962be151531960c1 2024-11-22T17:17:24,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/9374d999d9d549fcbbe7cdc295468931 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/9374d999d9d549fcbbe7cdc295468931 2024-11-22T17:17:24,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c5ab81e40903428ca7af0b94844eb302 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c5ab81e40903428ca7af0b94844eb302 2024-11-22T17:17:24,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f1648f4f25b6477da20a053153872a38 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/f1648f4f25b6477da20a053153872a38 2024-11-22T17:17:24,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/05ba312a9cae483aa268535cf5025127 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/05ba312a9cae483aa268535cf5025127 2024-11-22T17:17:24,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3f9f654701d748918e738664fa8134c5 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/3f9f654701d748918e738664fa8134c5 2024-11-22T17:17:24,768 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/recovered.edits/407.seqid, newMaxSeqId=407, maxSeqId=4 2024-11-22T17:17:24,768 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. 2024-11-22T17:17:24,769 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for 83841b437d25c9378a29982f12a6cd16: Waiting for close lock at 1732295843376Running coprocessor pre-close hooks at 1732295843376Disabling compacts and flushes for region at 1732295843376Disabling writes for close at 1732295843451 (+75 ms)Obtaining lock to block concurrent updates at 1732295843451Preparing flush snapshotting stores in 83841b437d25c9378a29982f12a6cd16 at 1732295843451Finished memstore snapshotting TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732295843451Flushing stores of TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16. at 1732295843452 (+1 ms)Flushing 83841b437d25c9378a29982f12a6cd16/A: creating writer at 1732295843452Flushing mob file 83841b437d25c9378a29982f12a6cd16/A: appending metadata at 1732295843455 (+3 ms)Flushing mob file 83841b437d25c9378a29982f12a6cd16/A: closing flushed file at 1732295843455Flushing 83841b437d25c9378a29982f12a6cd16/A: appending metadata at 1732295843862 (+407 ms)Flushing 83841b437d25c9378a29982f12a6cd16/A: closing flushed file at 1732295843862Flushing 83841b437d25c9378a29982f12a6cd16/B: creating writer at 1732295844268 (+406 ms)Flushing 83841b437d25c9378a29982f12a6cd16/B: appending metadata at 1732295844270 (+2 ms)Flushing 83841b437d25c9378a29982f12a6cd16/B: closing flushed file at 1732295844270Flushing 83841b437d25c9378a29982f12a6cd16/C: creating writer at 1732295844675 (+405 ms)Flushing 83841b437d25c9378a29982f12a6cd16/C: appending metadata at 1732295844677 (+2 ms)Flushing 83841b437d25c9378a29982f12a6cd16/C: closing flushed file at 1732295844677Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2eb83bd6: reopening flushed file at 1732295844683 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@163185b9: reopening flushed file at 1732295844686 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b9ab0c4: reopening flushed file at 1732295844688 (+2 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 83841b437d25c9378a29982f12a6cd16 in 1240ms, sequenceid=404, compaction requested=true at 1732295844691 (+3 ms)Writing region close event to WAL at 1732295844766 (+75 ms)Running coprocessor post-close hooks at 1732295844768 (+2 ms)Closed at 1732295844768 2024-11-22T17:17:24,770 INFO [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed 83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:24,770 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=83841b437d25c9378a29982f12a6cd16, regionState=CLOSED 2024-11-22T17:17:24,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 because future has completed 2024-11-22T17:17:24,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-22T17:17:24,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure 83841b437d25c9378a29982f12a6cd16, server=e61df7af61d2,42253,1732295655623 in 1.5480 sec 2024-11-22T17:17:24,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-22T17:17:24,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83841b437d25c9378a29982f12a6cd16, UNASSIGN in 1.5520 sec 2024-11-22T17:17:24,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-22T17:17:24,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5540 sec 2024-11-22T17:17:24,776 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732295844776"}]},"ts":"1732295844776"} 2024-11-22T17:17:24,777 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T17:17:24,777 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T17:17:24,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5620 sec 2024-11-22T17:17:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T17:17:25,372 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:17:25,372 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T17:17:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:17:25,373 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:17:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T17:17:25,374 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:17:25,376 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,378 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C, FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/recovered.edits] 2024-11-22T17:17:25,380 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/29564308669446ce9eb7d56f5986447c to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/29564308669446ce9eb7d56f5986447c 2024-11-22T17:17:25,381 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/3a8733fa59f84e4bbbff1f262ffd250b to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/3a8733fa59f84e4bbbff1f262ffd250b 2024-11-22T17:17:25,382 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a26345a11410456c8114e765549f256e to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/A/a26345a11410456c8114e765549f256e 2024-11-22T17:17:25,383 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/edb7dc21793746788fbaf72fa58c05dd to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/edb7dc21793746788fbaf72fa58c05dd 2024-11-22T17:17:25,384 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/f414652f2a4449b18100a4883a07b53a to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/f414652f2a4449b18100a4883a07b53a 2024-11-22T17:17:25,385 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/fc19c1db8cad42c994dcd0b1a777ddce to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/B/fc19c1db8cad42c994dcd0b1a777ddce 2024-11-22T17:17:25,386 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/1c6c4967a35c491ba4e64dca9fde2d57 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/1c6c4967a35c491ba4e64dca9fde2d57 2024-11-22T17:17:25,387 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/7381c2104d6d4759a644fbde45a2d0a8 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/7381c2104d6d4759a644fbde45a2d0a8 2024-11-22T17:17:25,388 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c003ff907f464df28a5905f4bad94903 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/C/c003ff907f464df28a5905f4bad94903 2024-11-22T17:17:25,389 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/recovered.edits/407.seqid to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16/recovered.edits/407.seqid 2024-11-22T17:17:25,389 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/default/TestAcidGuarantees/83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,389 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T17:17:25,390 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T17:17:25,390 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T17:17:25,392 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220cfe7b00c79d4042bd5bc792c370aadf_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220cfe7b00c79d4042bd5bc792c370aadf_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,393 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220d8b9f3fc03b4722a12ae98c5ff33780_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220d8b9f3fc03b4722a12ae98c5ff33780_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,393 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220eaa77a5cc5f4afc957a5c50904d43a1_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220eaa77a5cc5f4afc957a5c50904d43a1_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,394 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112222b912b1ce8b4d3299f1dded4f1bccee_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112222b912b1ce8b4d3299f1dded4f1bccee_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,395 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223ac254d1a402429bb359471cc0f3d5fb_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223ac254d1a402429bb359471cc0f3d5fb_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,395 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223f3371a626c744e0b2638f84395c16bb_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223f3371a626c744e0b2638f84395c16bb_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,396 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223f5ffaf139cc445290f9f6216e41ea46_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223f5ffaf139cc445290f9f6216e41ea46_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,396 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112250c2487443a34e31a1ce06fb8a93ab41_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112250c2487443a34e31a1ce06fb8a93ab41_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,397 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112272b3739a74824d97905dba9ae95af1f0_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112272b3739a74824d97905dba9ae95af1f0_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,398 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227d4a1a24b9294cdeb04db5e170610493_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227d4a1a24b9294cdeb04db5e170610493_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,398 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227e028edcdd614e6c9e469b832c4b039b_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227e028edcdd614e6c9e469b832c4b039b_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,399 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227edba3b0020f487e8b770c02ebac8572_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227edba3b0020f487e8b770c02ebac8572_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,400 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112293efde1b0fef423b84ae749c28ccb74a_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112293efde1b0fef423b84ae749c28ccb74a_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,400 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a2e7cfd75f9349f8aa5b8a6b983fcfb2_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a2e7cfd75f9349f8aa5b8a6b983fcfb2_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,401 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a4d8fb7850204839ae5bf71dcecfb3fe_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a4d8fb7850204839ae5bf71dcecfb3fe_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,401 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a5e0f6f13d1f44d2b8e1c5966b820a17_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a5e0f6f13d1f44d2b8e1c5966b820a17_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,402 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bc84c76ee90e47969009f97173ede769_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bc84c76ee90e47969009f97173ede769_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,403 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bdb442c650c44c80ab860d9f77538d84_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bdb442c650c44c80ab860d9f77538d84_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,403 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ce46211b41624c138cebaa67f375395d_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ce46211b41624c138cebaa67f375395d_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,404 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d3098268c6d94c99baacd8c571f7ea1f_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d3098268c6d94c99baacd8c571f7ea1f_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,405 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e139d6463686472495cb5793fe5ddd2b_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e139d6463686472495cb5793fe5ddd2b_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,405 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fde12bdcec0f4f61ab9610f431978420_83841b437d25c9378a29982f12a6cd16 to hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fde12bdcec0f4f61ab9610f431978420_83841b437d25c9378a29982f12a6cd16 2024-11-22T17:17:25,406 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T17:17:25,407 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:17:25,409 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T17:17:25,410 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T17:17:25,411 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:17:25,411 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T17:17:25,411 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732295845411"}]},"ts":"9223372036854775807"} 2024-11-22T17:17:25,413 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T17:17:25,413 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 83841b437d25c9378a29982f12a6cd16, NAME => 'TestAcidGuarantees,,1732295818570.83841b437d25c9378a29982f12a6cd16.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T17:17:25,413 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T17:17:25,413 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732295845413"}]},"ts":"9223372036854775807"} 2024-11-22T17:17:25,414 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T17:17:25,414 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T17:17:25,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-22T17:17:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T17:17:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T17:17:25,482 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T17:17:25,489 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252 (was 249) - Thread LEAK? -, OpenFileDescriptor=457 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=507 (was 429) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2105 (was 2606) 2024-11-22T17:17:25,489 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T17:17:25,489 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T17:17:25,489 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T17:17:25,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:25,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:25,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T17:17:25,489 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T17:17:25,490 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=694036216, stopped=false 2024-11-22T17:17:25,490 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e61df7af61d2,45693,1732295654407 2024-11-22T17:17:25,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T17:17:25,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T17:17:25,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:17:25,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:17:25,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T17:17:25,596 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T17:17:25,596 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T17:17:25,596 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T17:17:25,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:25,596 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e61df7af61d2,42253,1732295655623' ***** 2024-11-22T17:17:25,596 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T17:17:25,596 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T17:17:25,596 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T17:17:25,597 INFO [RS:0;e61df7af61d2:42253 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T17:17:25,597 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T17:17:25,597 INFO [RS:0;e61df7af61d2:42253 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T17:17:25,597 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(959): stopping server e61df7af61d2,42253,1732295655623 2024-11-22T17:17:25,597 INFO [RS:0;e61df7af61d2:42253 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T17:17:25,597 INFO [RS:0;e61df7af61d2:42253 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e61df7af61d2:42253. 2024-11-22T17:17:25,597 DEBUG [RS:0;e61df7af61d2:42253 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T17:17:25,597 DEBUG [RS:0;e61df7af61d2:42253 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:25,597 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T17:17:25,597 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T17:17:25,597 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T17:17:25,597 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T17:17:25,598 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T17:17:25,598 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-22T17:17:25,598 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T17:17:25,599 INFO [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T17:17:25,599 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T17:17:25,599 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T17:17:25,599 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T17:17:25,599 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T17:17:25,599 INFO [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-22T17:17:25,621 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/.tmp/info/5adae09231c149c1863354543a8811a3 is 95, key is TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5./info:/1732295691824/DeleteFamily/seqid=0 2024-11-22T17:17:25,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742445_1621 (size=6514) 2024-11-22T17:17:25,642 INFO [regionserver/e61df7af61d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T17:17:25,799 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T17:17:25,942 INFO [regionserver/e61df7af61d2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-22T17:17:25,943 INFO [regionserver/e61df7af61d2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-22T17:17:25,999 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T17:17:26,025 INFO [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/.tmp/info/5adae09231c149c1863354543a8811a3 2024-11-22T17:17:26,042 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/.tmp/ns/344c487159dc406890f31fe16d2cd398 is 93, key is TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5./ns:/1732295691782/DeleteFamily/seqid=0 2024-11-22T17:17:26,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742446_1622 (size=5906) 2024-11-22T17:17:26,199 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T17:17:26,399 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T17:17:26,445 INFO [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/.tmp/ns/344c487159dc406890f31fe16d2cd398 2024-11-22T17:17:26,460 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/.tmp/rep_barrier/638300be49a84416946ecfe1d97a733a is 102, key is TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5./rep_barrier:/1732295691782/DeleteFamily/seqid=0 2024-11-22T17:17:26,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742447_1623 (size=6025) 2024-11-22T17:17:26,599 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T17:17:26,800 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T17:17:26,864 INFO [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/.tmp/rep_barrier/638300be49a84416946ecfe1d97a733a 2024-11-22T17:17:26,880 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/.tmp/table/7763516500084444b5c88ae24e872438 is 96, key is TestAcidGuarantees,,1732295659249.c70a0885ec0e4f41398ae6890c8dabe5./table:/1732295691782/DeleteFamily/seqid=0 2024-11-22T17:17:26,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742448_1624 (size=5936) 2024-11-22T17:17:27,000 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T17:17:27,200 DEBUG [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T17:17:27,283 INFO [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/.tmp/table/7763516500084444b5c88ae24e872438 2024-11-22T17:17:27,287 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/.tmp/info/5adae09231c149c1863354543a8811a3 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/info/5adae09231c149c1863354543a8811a3 2024-11-22T17:17:27,289 INFO [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/info/5adae09231c149c1863354543a8811a3, entries=12, sequenceid=90, filesize=6.4 K 2024-11-22T17:17:27,289 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/.tmp/ns/344c487159dc406890f31fe16d2cd398 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/ns/344c487159dc406890f31fe16d2cd398 2024-11-22T17:17:27,292 INFO [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/ns/344c487159dc406890f31fe16d2cd398, entries=8, sequenceid=90, filesize=5.8 K 2024-11-22T17:17:27,292 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/.tmp/rep_barrier/638300be49a84416946ecfe1d97a733a as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/rep_barrier/638300be49a84416946ecfe1d97a733a 2024-11-22T17:17:27,294 INFO [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/rep_barrier/638300be49a84416946ecfe1d97a733a, entries=6, sequenceid=90, filesize=5.9 K 2024-11-22T17:17:27,295 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/.tmp/table/7763516500084444b5c88ae24e872438 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/table/7763516500084444b5c88ae24e872438 2024-11-22T17:17:27,297 INFO [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/table/7763516500084444b5c88ae24e872438, entries=7, sequenceid=90, filesize=5.8 K 2024-11-22T17:17:27,297 INFO [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1698ms, sequenceid=90, compaction requested=false 2024-11-22T17:17:27,301 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-22T17:17:27,301 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T17:17:27,301 INFO [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T17:17:27,301 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732295845598Running coprocessor pre-close hooks at 1732295845598Disabling compacts and flushes for region at 1732295845598Disabling writes for close at 1732295845599 (+1 ms)Obtaining lock to block concurrent updates at 1732295845599Preparing flush snapshotting stores in 1588230740 at 1732295845599Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732295845599Flushing stores of hbase:meta,,1.1588230740 at 1732295845600 (+1 ms)Flushing 1588230740/info: creating writer at 1732295845600Flushing 1588230740/info: appending metadata at 1732295845621 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732295845621Flushing 1588230740/ns: creating writer at 1732295846029 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732295846042 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732295846042Flushing 1588230740/rep_barrier: creating writer at 1732295846448 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732295846460 (+12 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732295846460Flushing 1588230740/table: creating writer at 1732295846866 (+406 ms)Flushing 1588230740/table: appending metadata at 1732295846880 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732295846880Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@730a6df8: reopening flushed file at 1732295847286 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f540bf4: reopening flushed file at 1732295847289 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70ee3e59: reopening flushed file at 1732295847292 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e162daa: reopening flushed file at 1732295847294 (+2 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1698ms, sequenceid=90, compaction requested=false at 1732295847297 (+3 ms)Writing region close event to WAL at 1732295847299 (+2 ms)Running coprocessor post-close hooks at 1732295847301 (+2 ms)Closed at 1732295847301 2024-11-22T17:17:27,301 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T17:17:27,400 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(976): stopping server e61df7af61d2,42253,1732295655623; all regions closed. 2024-11-22T17:17:27,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741834_1010 (size=25344) 2024-11-22T17:17:27,407 DEBUG [RS:0;e61df7af61d2:42253 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/oldWALs 2024-11-22T17:17:27,407 INFO [RS:0;e61df7af61d2:42253 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e61df7af61d2%2C42253%2C1732295655623.meta:.meta(num 1732295658702) 2024-11-22T17:17:27,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741833_1009 (size=16495760) 2024-11-22T17:17:27,418 DEBUG [RS:0;e61df7af61d2:42253 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/oldWALs 2024-11-22T17:17:27,418 INFO [RS:0;e61df7af61d2:42253 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e61df7af61d2%2C42253%2C1732295655623:(num 1732295658078) 2024-11-22T17:17:27,418 DEBUG [RS:0;e61df7af61d2:42253 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T17:17:27,418 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T17:17:27,418 INFO [RS:0;e61df7af61d2:42253 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T17:17:27,418 INFO [RS:0;e61df7af61d2:42253 {}] hbase.ChoreService(370): Chore service for: regionserver/e61df7af61d2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T17:17:27,418 INFO [RS:0;e61df7af61d2:42253 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T17:17:27,419 INFO [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T17:17:27,419 INFO [RS:0;e61df7af61d2:42253 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42253 2024-11-22T17:17:27,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e61df7af61d2,42253,1732295655623 2024-11-22T17:17:27,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T17:17:27,434 INFO [RS:0;e61df7af61d2:42253 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T17:17:27,446 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e61df7af61d2,42253,1732295655623] 2024-11-22T17:17:27,456 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e61df7af61d2,42253,1732295655623 already deleted, retry=false 2024-11-22T17:17:27,457 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e61df7af61d2,42253,1732295655623 expired; onlineServers=0 2024-11-22T17:17:27,457 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e61df7af61d2,45693,1732295654407' ***** 2024-11-22T17:17:27,457 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T17:17:27,457 INFO [M:0;e61df7af61d2:45693 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T17:17:27,457 INFO [M:0;e61df7af61d2:45693 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T17:17:27,457 DEBUG [M:0;e61df7af61d2:45693 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T17:17:27,457 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T17:17:27,457 DEBUG [M:0;e61df7af61d2:45693 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T17:17:27,457 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732295657788 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732295657788,5,FailOnTimeoutGroup] 2024-11-22T17:17:27,457 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732295657789 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732295657789,5,FailOnTimeoutGroup] 2024-11-22T17:17:27,457 INFO [M:0;e61df7af61d2:45693 {}] hbase.ChoreService(370): Chore service for: master/e61df7af61d2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T17:17:27,457 INFO [M:0;e61df7af61d2:45693 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T17:17:27,458 DEBUG [M:0;e61df7af61d2:45693 {}] master.HMaster(1795): Stopping service threads 2024-11-22T17:17:27,458 INFO [M:0;e61df7af61d2:45693 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T17:17:27,458 INFO [M:0;e61df7af61d2:45693 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T17:17:27,458 ERROR [M:0;e61df7af61d2:45693 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1297744652) connection to localhost/127.0.0.1:40221 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:40221,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-22T17:17:27,459 INFO [M:0;e61df7af61d2:45693 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T17:17:27,459 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T17:17:27,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T17:17:27,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T17:17:27,467 DEBUG [M:0;e61df7af61d2:45693 {}] zookeeper.ZKUtil(347): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T17:17:27,467 WARN [M:0;e61df7af61d2:45693 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T17:17:27,468 INFO [M:0;e61df7af61d2:45693 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/.lastflushedseqids 2024-11-22T17:17:27,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742449_1625 (size=72) 2024-11-22T17:17:27,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T17:17:27,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42253-0x1016389f4780001, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T17:17:27,547 INFO [RS:0;e61df7af61d2:42253 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T17:17:27,547 INFO [RS:0;e61df7af61d2:42253 {}] regionserver.HRegionServer(1031): Exiting; stopping=e61df7af61d2,42253,1732295655623; zookeeper connection closed. 2024-11-22T17:17:27,547 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@68456424 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@68456424 2024-11-22T17:17:27,548 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-22T17:17:27,876 INFO [M:0;e61df7af61d2:45693 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T17:17:27,877 INFO [M:0;e61df7af61d2:45693 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T17:17:27,877 DEBUG [M:0;e61df7af61d2:45693 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T17:17:27,877 INFO [M:0;e61df7af61d2:45693 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T17:17:27,877 DEBUG [M:0;e61df7af61d2:45693 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T17:17:27,877 DEBUG [M:0;e61df7af61d2:45693 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T17:17:27,877 DEBUG [M:0;e61df7af61d2:45693 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T17:17:27,877 INFO [M:0;e61df7af61d2:45693 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=784.68 KB heapSize=962.07 KB 2024-11-22T17:17:27,891 DEBUG [M:0;e61df7af61d2:45693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8a6c31cc72e1465094bdd23084bfd898 is 82, key is hbase:meta,,1/info:regioninfo/1732295658782/Put/seqid=0 2024-11-22T17:17:27,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742450_1626 (size=5672) 2024-11-22T17:17:28,295 INFO [M:0;e61df7af61d2:45693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2200 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8a6c31cc72e1465094bdd23084bfd898 2024-11-22T17:17:28,314 DEBUG [M:0;e61df7af61d2:45693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bfed30bba794437ab05ed55429a86201 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x1F/proc:d/1732295695008/Put/seqid=0 2024-11-22T17:17:28,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742451_1627 (size=44381) 2024-11-22T17:17:28,718 INFO [M:0;e61df7af61d2:45693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=784.13 KB at sequenceid=2200 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bfed30bba794437ab05ed55429a86201 2024-11-22T17:17:28,721 INFO [M:0;e61df7af61d2:45693 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bfed30bba794437ab05ed55429a86201 2024-11-22T17:17:28,735 DEBUG [M:0;e61df7af61d2:45693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/59853b4c4db54d6d927decc3cb57b081 is 69, key is e61df7af61d2,42253,1732295655623/rs:state/1732295657839/Put/seqid=0 2024-11-22T17:17:28,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073742452_1628 (size=5156) 2024-11-22T17:17:29,138 INFO [M:0;e61df7af61d2:45693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2200 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/59853b4c4db54d6d927decc3cb57b081 2024-11-22T17:17:29,142 DEBUG [M:0;e61df7af61d2:45693 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8a6c31cc72e1465094bdd23084bfd898 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8a6c31cc72e1465094bdd23084bfd898 2024-11-22T17:17:29,145 INFO [M:0;e61df7af61d2:45693 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8a6c31cc72e1465094bdd23084bfd898, entries=8, sequenceid=2200, filesize=5.5 K 2024-11-22T17:17:29,146 DEBUG [M:0;e61df7af61d2:45693 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bfed30bba794437ab05ed55429a86201 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bfed30bba794437ab05ed55429a86201 2024-11-22T17:17:29,149 INFO [M:0;e61df7af61d2:45693 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bfed30bba794437ab05ed55429a86201 2024-11-22T17:17:29,149 INFO [M:0;e61df7af61d2:45693 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bfed30bba794437ab05ed55429a86201, entries=164, sequenceid=2200, filesize=43.3 K 2024-11-22T17:17:29,150 DEBUG [M:0;e61df7af61d2:45693 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/59853b4c4db54d6d927decc3cb57b081 as hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/59853b4c4db54d6d927decc3cb57b081 2024-11-22T17:17:29,152 INFO [M:0;e61df7af61d2:45693 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/c525cc13-be9f-f4bb-6875-af9afb8049b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/59853b4c4db54d6d927decc3cb57b081, entries=1, sequenceid=2200, filesize=5.0 K 2024-11-22T17:17:29,153 INFO [M:0;e61df7af61d2:45693 {}] regionserver.HRegion(3140): Finished flush of dataSize ~784.68 KB/803515, heapSize ~961.77 KB/984856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2200, compaction requested=false 2024-11-22T17:17:29,154 INFO [M:0;e61df7af61d2:45693 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T17:17:29,154 DEBUG [M:0;e61df7af61d2:45693 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732295847877Disabling compacts and flushes for region at 1732295847877Disabling writes for close at 1732295847877Obtaining lock to block concurrent updates at 1732295847877Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732295847877Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=803515, getHeapSize=985096, getOffHeapSize=0, getCellsCount=2389 at 1732295847877Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732295847878 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732295847878Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732295847891 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732295847891Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732295848298 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732295848314 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732295848314Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732295848721 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732295848734 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732295848735 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36c2b8b9: reopening flushed file at 1732295849141 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d4c7ec5: reopening flushed file at 1732295849145 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72c64731: reopening flushed file at 1732295849149 (+4 ms)Finished flush of dataSize ~784.68 KB/803515, heapSize ~961.77 KB/984856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2200, compaction requested=false at 1732295849153 (+4 ms)Writing region close event to WAL at 1732295849154 (+1 ms)Closed at 1732295849154 2024-11-22T17:17:29,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35457 is added to blk_1073741830_1006 (size=947690) 2024-11-22T17:17:29,157 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T17:17:29,157 INFO [M:0;e61df7af61d2:45693 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T17:17:29,157 INFO [M:0;e61df7af61d2:45693 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45693 2024-11-22T17:17:29,157 INFO [M:0;e61df7af61d2:45693 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T17:17:29,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T17:17:29,262 INFO [M:0;e61df7af61d2:45693 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T17:17:29,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45693-0x1016389f4780000, quorum=127.0.0.1:64218, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T17:17:29,266 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@72835e42{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T17:17:29,268 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2098bdc7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T17:17:29,269 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T17:17:29,269 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T17:17:29,269 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4828e07b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/hadoop.log.dir/,STOPPED} 2024-11-22T17:17:29,272 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-22T17:17:29,272 WARN [BP-932480829-172.17.0.2-1732295650022 heartbeating to localhost/127.0.0.1:40221 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-22T17:17:29,272 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-22T17:17:29,272 WARN [BP-932480829-172.17.0.2-1732295650022 heartbeating to localhost/127.0.0.1:40221 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-932480829-172.17.0.2-1732295650022 (Datanode Uuid 355a9353-cd06-4c0f-a5c5-96dfa7221a50) service to localhost/127.0.0.1:40221 2024-11-22T17:17:29,274 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/cluster_c396a244-2610-d741-083f-3e053f26e882/data/data1/current/BP-932480829-172.17.0.2-1732295650022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T17:17:29,275 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/cluster_c396a244-2610-d741-083f-3e053f26e882/data/data2/current/BP-932480829-172.17.0.2-1732295650022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T17:17:29,275 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-22T17:17:29,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T17:17:29,282 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T17:17:29,282 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T17:17:29,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T17:17:29,283 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8bb41bfd-428e-b993-fc72-1297f6ffa374/hadoop.log.dir/,STOPPED} 2024-11-22T17:17:29,301 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T17:17:29,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down